From swift at ci.uchicago.edu Thu Jul 3 12:10:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Thu, 3 Jul 2014 12:10:03 -0500 (CDT) Subject: [Swift-commit] cog r3994 Message-ID: <20140703171003.ADB928D000A9@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3994 | timgarmstrong | 2014-07-03 12:09:45 -0500 (Thu, 03 Jul 2014) | 1 line Basic parsing for coaster settings with escapes and quotes ------------------------------------------------------------------------ Index: modules/provider-coaster-c-client/src/Settings.h =================================================================== --- modules/provider-coaster-c-client/src/Settings.h (revision 3993) +++ modules/provider-coaster-c-client/src/Settings.h (working copy) @@ -34,7 +34,16 @@ void set(const char* key, size_t key_len, const char* value, size_t value_len); void remove(const std::string& key); + + bool contains(const std::string& key); + /* + * Get key from map + * returns true if key exists, false if doesn't + * value: set to value if key exists + */ + bool get(const std::string& key, std::string &value); + std::map& getSettings(); template friend cls& operator<< (cls& os, Settings& s); Index: modules/provider-coaster-c-client/src/coasters.cpp =================================================================== --- modules/provider-coaster-c-client/src/coasters.cpp (revision 3993) +++ modules/provider-coaster-c-client/src/coasters.cpp (working copy) @@ -127,14 +127,96 @@ } } -coaster_rc coaster_settings_parse(coaster_settings *settings, - const char *str, size_t str_len) +static void +settings_emit(coaster_settings *settings, string &key, string &value); + +coaster_rc +coaster_settings_parse(coaster_settings *settings, + const char *str, size_t str_len, char separator) COASTER_THROWS_NOTHING { + COASTER_CONDITION(settings != NULL, COASTER_ERROR_INVALID, + "Null Coaster settings object"); + COASTER_CONDITION(str != NULL, COASTER_ERROR_INVALID, + "Null Coaster settings string"); + string key, value; // Storage for current key and value + + bool in_key = true; // Either in key or value + bool in_quotes = false; + bool in_escape = false; - // TODO: parsing using code currently in CoasterSwig - return COASTER_ERROR_UNKNOWN; + for (size_t pos = 0; pos < str_len; pos++) { + char c = str[pos]; + string &curr = in_key ? key : value; + + if (in_escape) { + // Current character is escaped + curr.push_back(c); + in_escape = false; + + } else if (in_quotes) { + if (c == '\\') { + in_escape = true; + } else if (c == '"') { + in_quotes = false; + } else { + curr.push_back(c); + } + } else { + // Not in escape or quotes + if (c == '\\') { + in_escape = true; + } else if (c == '"') { + in_quotes = true; + } else if (c == '=') { + COASTER_CONDITION(in_key, COASTER_ERROR_INVALID, + "'=' not allowed in unquoted Coaster settings value"); + in_key = false; + } else if (c == separator) { + COASTER_CONDITION(!in_key, COASTER_ERROR_INVALID, + "',' not allowed in unquoted Coaster settings key"); + + settings_emit(settings, key, value); + + in_key = true; + } else { + curr.push_back(c); + } + } + } + + // Check for invalid state + COASTER_CONDITION(!in_escape, COASTER_ERROR_INVALID, + "Trailing '\\' escape at end of Coaster settings"); + COASTER_CONDITION(!in_quotes, COASTER_ERROR_INVALID, + "Unclosed '\"' quote at end of Coaster settings"); + COASTER_CONDITION(!in_key, COASTER_ERROR_INVALID, + "Key without value at end of Coaster settings"); + + settings_emit(settings, key, value); + + return COASTER_SUCCESS; } + +/* + * Helper function to emit completed key/value setting + */ +static void +settings_emit(coaster_settings *settings, string &key, string &value) { + if (settings->contains(key)) { + string old_value; + settings->get(key, old_value); + LogWarn << "Overwrote previous Coaster settings value for " + "key: \"" << key << "\". Old value: \"" << + old_value << "\", New value: \"" << + value << "\"." << endl; + } + + settings->set(key, value); + key.clear(); + value.clear(); +} + coaster_rc coaster_settings_set(coaster_settings *settings, const char *key, size_t key_len, Index: modules/provider-coaster-c-client/src/Settings.cpp =================================================================== --- modules/provider-coaster-c-client/src/Settings.cpp (revision 3993) +++ modules/provider-coaster-c-client/src/Settings.cpp (working copy) @@ -60,6 +60,20 @@ settings.erase(key); } +bool Settings::contains(const string& key) { + return settings.find(key) != settings.end(); +} + +bool Settings::get(const string& key, string& value) { + map::iterator it = settings.find(key); + if (settings.find(key) == settings.end()) { + return false; + } else { + value = it->second; + return true; + } +} + map& Settings::getSettings() { return settings; } Index: modules/provider-coaster-c-client/src/coasters.h =================================================================== --- modules/provider-coaster-c-client/src/coasters.h (revision 3993) +++ modules/provider-coaster-c-client/src/coasters.h (working copy) @@ -134,12 +134,14 @@ * Parse settings from string. * * str[_len]: String with key/value settings and length of string. - Settings separated by commas, key/values separated by equals sign. - If NULL, will create empty settings object. + Settings separated by separator char, key/values separated by equals sign. + Backslash escapes the next character. + Keys and values can be quoted with ". + separator: character to separate keys/values */ coaster_rc coaster_settings_parse(coaster_settings *settings, const char *str, - size_t str_len) COASTER_THROWS_NOTHING; + size_t str_len, char separator) COASTER_THROWS_NOTHING; /* * Set settings individually. */ From hategan at ci.uchicago.edu Thu Jul 3 20:26:18 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 3 Jul 2014 20:26:18 -0500 (CDT) Subject: [Swift-commit] r7934 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20140704012618.D673B9D9BC@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-03 20:26:17 -0500 (Thu, 03 Jul 2014) New Revision: 7934 Modified: trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java Log: Fixed compilation of param files on windows Modified: trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-06-28 18:52:46 UTC (rev 7933) +++ trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-07-04 01:26:17 UTC (rev 7934) @@ -104,7 +104,8 @@ } br.close(); - String pkg = f.getParentFile().getAbsolutePath().substring(base.getAbsolutePath().length() + 1).replace('/', '.'); + String pkg = f.getParentFile().getAbsolutePath().substring( + base.getAbsolutePath().length() + 1).replace(File.pathSeparatorChar, '.'); File nf = new File(makeFileName(f)); writeFile(nf, pkg, params, opts); } From hategan at ci.uchicago.edu Thu Jul 3 20:29:03 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 3 Jul 2014 20:29:03 -0500 (CDT) Subject: [Swift-commit] r7935 - branches/release-0.95/src/org/griphyn/vdl/mapping/file Message-ID: <20140704012903.B2CFE9D9BC@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-03 20:29:03 -0500 (Thu, 03 Jul 2014) New Revision: 7935 Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java Log: Fixed compilation of param files on windows Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java =================================================================== --- branches/release-0.95/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-07-04 01:26:17 UTC (rev 7934) +++ branches/release-0.95/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-07-04 01:29:03 UTC (rev 7935) @@ -104,7 +104,8 @@ } br.close(); - String pkg = f.getParentFile().getAbsolutePath().substring(base.getAbsolutePath().length() + 1).replace('/', '.'); + String pkg = f.getParentFile().getAbsolutePath().substring( + base.getAbsolutePath().length() + 1).replace(File.pathSeparatorChar, '.'); File nf = new File(makeFileName(f)); writeFile(nf, pkg, params, opts); } From hategan at ci.uchicago.edu Thu Jul 3 20:35:28 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 3 Jul 2014 20:35:28 -0500 (CDT) Subject: [Swift-commit] r7936 - trunk/src/org/griphyn/vdl/mapping/file Message-ID: <20140704013528.A72AF9D9BC@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-03 20:35:28 -0500 (Thu, 03 Jul 2014) New Revision: 7936 Modified: trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java Log: Ooops, File.separatorChar, not File.pathSeparatorChar Modified: trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-07-04 01:29:03 UTC (rev 7935) +++ trunk/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-07-04 01:35:28 UTC (rev 7936) @@ -105,7 +105,7 @@ br.close(); String pkg = f.getParentFile().getAbsolutePath().substring( - base.getAbsolutePath().length() + 1).replace(File.pathSeparatorChar, '.'); + base.getAbsolutePath().length() + 1).replace(File.separatorChar, '.'); File nf = new File(makeFileName(f)); writeFile(nf, pkg, params, opts); } From hategan at ci.uchicago.edu Thu Jul 3 20:36:39 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 3 Jul 2014 20:36:39 -0500 (CDT) Subject: [Swift-commit] r7937 - branches/release-0.95/src/org/griphyn/vdl/mapping/file Message-ID: <20140704013639.6A3529D9BC@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-03 20:36:39 -0500 (Thu, 03 Jul 2014) New Revision: 7937 Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java Log: ... Modified: branches/release-0.95/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java =================================================================== --- branches/release-0.95/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-07-04 01:35:28 UTC (rev 7936) +++ branches/release-0.95/src/org/griphyn/vdl/mapping/file/MappingParamFileGenerator.java 2014-07-04 01:36:39 UTC (rev 7937) @@ -105,7 +105,7 @@ br.close(); String pkg = f.getParentFile().getAbsolutePath().substring( - base.getAbsolutePath().length() + 1).replace(File.pathSeparatorChar, '.'); + base.getAbsolutePath().length() + 1).replace(File.separatorChar, '.'); File nf = new File(makeFileName(f)); writeFile(nf, pkg, params, opts); } From hategan at ci.uchicago.edu Thu Jul 3 21:02:24 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 3 Jul 2014 21:02:24 -0500 (CDT) Subject: [Swift-commit] r7938 - trunk/libexec Message-ID: <20140704020224.DE57D9D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-03 21:02:24 -0500 (Thu, 03 Jul 2014) New Revision: 7938 Modified: trunk/libexec/_swiftwrap.vbs Log: fixed swiftwrap on windows (removed kickstart argument) Modified: trunk/libexec/_swiftwrap.vbs =================================================================== --- trunk/libexec/_swiftwrap.vbs 2014-07-04 01:36:39 UTC (rev 7937) +++ trunk/libexec/_swiftwrap.vbs 2014-07-04 02:02:24 UTC (rev 7938) @@ -184,9 +184,6 @@ expectArg("of") OUTF=getOptArg() -expectArg("k") -KICKSTART=getOptArg() - expectArg("cdmfile") 'ignored, but read if specified CDMFILE=getOptArg() From hategan at ci.uchicago.edu Thu Jul 3 21:03:57 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 3 Jul 2014 21:03:57 -0500 (CDT) Subject: [Swift-commit] r7939 - branches/release-0.95/libexec Message-ID: <20140704020357.C63649D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-03 21:03:57 -0500 (Thu, 03 Jul 2014) New Revision: 7939 Modified: branches/release-0.95/libexec/_swiftwrap.vbs Log: fixed swiftwrap on windows (removed kickstart argument) Modified: branches/release-0.95/libexec/_swiftwrap.vbs =================================================================== --- branches/release-0.95/libexec/_swiftwrap.vbs 2014-07-04 02:02:24 UTC (rev 7938) +++ branches/release-0.95/libexec/_swiftwrap.vbs 2014-07-04 02:03:57 UTC (rev 7939) @@ -184,9 +184,6 @@ expectArg("of") OUTF=getOptArg() -expectArg("k") -KICKSTART=getOptArg() - expectArg("cdmfile") 'ignored, but read if specified CDMFILE=getOptArg() From hategan at ci.uchicago.edu Thu Jul 3 21:18:49 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 3 Jul 2014 21:18:49 -0500 (CDT) Subject: [Swift-commit] r7940 - branches/release-0.95/libexec Message-ID: <20140704021849.CF3359D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-03 21:18:49 -0500 (Thu, 03 Jul 2014) New Revision: 7940 Modified: branches/release-0.95/libexec/_swiftwrap.vbs Log: removed remaining references to kickstart from _swiftwrap.vbs Modified: branches/release-0.95/libexec/_swiftwrap.vbs =================================================================== --- branches/release-0.95/libexec/_swiftwrap.vbs 2014-07-04 02:03:57 UTC (rev 7939) +++ branches/release-0.95/libexec/_swiftwrap.vbs 2014-07-04 02:18:49 UTC (rev 7940) @@ -222,7 +222,6 @@ log "INF=" + INF log "OUTF=" + OUTF log "STATUSMODE=" + STATUSMODE -log "KICKSTART=" + KICKSTART log "ARGS=" + Join(ARGS) logstate "CREATE_JOBDIR" @@ -268,62 +267,58 @@ End If End If -If KICKSTART = "" Then - Set min = Nothing - Set mout = Nothing - Set merr = Nothing - If STDIN <> "" Then - Set min = fs.OpenTextFile(STDIN, 1, False) - End If - If STDOUT <> "" Then - Set mout = fs.OpenTextFile(STDOUT, 2, True) - End If - If STDERR <> "" Then - Set merr = fs.OpenTextFile(STDERR, 2, True) - End If - qargs = prepareArgs(ARGS) - log "Cmd: " + prepareOne(EXEC) + " " + qargs - Set p = shell.exec(prepareOne(EXEC) + " " + qargs) - log "Executable started" +Set min = Nothing +Set mout = Nothing +Set merr = Nothing +If STDIN <> "" Then + Set min = fs.OpenTextFile(STDIN, 1, False) +End If +If STDOUT <> "" Then + Set mout = fs.OpenTextFile(STDOUT, 2, True) +End If +If STDERR <> "" Then + Set merr = fs.OpenTextFile(STDERR, 2, True) +End If +qargs = prepareArgs(ARGS) +log "Cmd: " + prepareOne(EXEC) + " " + qargs +Set p = shell.exec(prepareOne(EXEC) + " " + qargs) +log "Executable started" - Do Until p.StdOut.AtEndOfStream and p.StdErr.AtEndOfStream and p.Status <> 0 - some = False - If Not min Is Nothing Then - l = min.ReadLine - p.StdIn.Write(l) - some = True - End If - If Not p.StdOut.AtEndOfStream Then - l = p.StdOut.ReadLine - If Not mout Is Nothing Then - mout.Write(l) - End If - some = True - End If - If Not p.StdErr.AtEndOfStream Then - l = p.StdErr.ReadLine - If Not merr Is Nothing Then - merr.Write(l) - End If - some = True - End If - WScript.Sleep(100) - Loop +Do Until p.StdOut.AtEndOfStream and p.StdErr.AtEndOfStream and p.Status <> 0 + some = False If Not min Is Nothing Then - min.close() + l = min.ReadLine + p.StdIn.Write(l) + some = True End If - If Not mout Is Nothing Then - mout.close() + If Not p.StdOut.AtEndOfStream Then + l = p.StdOut.ReadLine + If Not mout Is Nothing Then + mout.Write(l) + End If + some = True End If - If Not merr Is Nothing Then - merr.close() + If Not p.StdErr.AtEndOfStream Then + l = p.StdErr.ReadLine + If Not merr Is Nothing Then + merr.Write(l) + End If + some = True End If - If p.ExitCode <> 0 Then - fail "Exit code " + CStr(p.ExitCode), p.ExitCode - End If -Else - fail "Kickstart is not supported on Windows", 250 + WScript.Sleep(100) +Loop +If Not min Is Nothing Then + min.close() End If +If Not mout Is Nothing Then + mout.close() +End If +If Not merr Is Nothing Then + merr.close() +End If +If p.ExitCode <> 0 Then + fail "Exit code " + CStr(p.ExitCode), p.ExitCode +End If shell.CurrentDirectory = WFDIR From hategan at ci.uchicago.edu Thu Jul 3 21:22:01 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 3 Jul 2014 21:22:01 -0500 (CDT) Subject: [Swift-commit] r7941 - trunk/libexec Message-ID: <20140704022201.BCB209D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-03 21:22:01 -0500 (Thu, 03 Jul 2014) New Revision: 7941 Modified: trunk/libexec/_swiftwrap.vbs Log: removed remaining references to kickstart from _swiftwrap.vbs (merged from 0.95) Modified: trunk/libexec/_swiftwrap.vbs =================================================================== --- trunk/libexec/_swiftwrap.vbs 2014-07-04 02:18:49 UTC (rev 7940) +++ trunk/libexec/_swiftwrap.vbs 2014-07-04 02:22:01 UTC (rev 7941) @@ -222,7 +222,6 @@ log "INF=" + INF log "OUTF=" + OUTF log "STATUSMODE=" + STATUSMODE -log "KICKSTART=" + KICKSTART log "ARGS=" + Join(ARGS) logstate "CREATE_JOBDIR" @@ -268,62 +267,58 @@ End If End If -If KICKSTART = "" Then - Set min = Nothing - Set mout = Nothing - Set merr = Nothing - If STDIN <> "" Then - Set min = fs.OpenTextFile(STDIN, 1, False) - End If - If STDOUT <> "" Then - Set mout = fs.OpenTextFile(STDOUT, 2, True) - End If - If STDERR <> "" Then - Set merr = fs.OpenTextFile(STDERR, 2, True) - End If - qargs = prepareArgs(ARGS) - log "Cmd: " + prepareOne(EXEC) + " " + qargs - Set p = shell.exec(prepareOne(EXEC) + " " + qargs) - log "Executable started" +Set min = Nothing +Set mout = Nothing +Set merr = Nothing +If STDIN <> "" Then + Set min = fs.OpenTextFile(STDIN, 1, False) +End If +If STDOUT <> "" Then + Set mout = fs.OpenTextFile(STDOUT, 2, True) +End If +If STDERR <> "" Then + Set merr = fs.OpenTextFile(STDERR, 2, True) +End If +qargs = prepareArgs(ARGS) +log "Cmd: " + prepareOne(EXEC) + " " + qargs +Set p = shell.exec(prepareOne(EXEC) + " " + qargs) +log "Executable started" - Do Until p.StdOut.AtEndOfStream and p.StdErr.AtEndOfStream and p.Status <> 0 - some = False - If Not min Is Nothing Then - l = min.ReadLine - p.StdIn.Write(l) - some = True - End If - If Not p.StdOut.AtEndOfStream Then - l = p.StdOut.ReadLine - If Not mout Is Nothing Then - mout.Write(l) - End If - some = True - End If - If Not p.StdErr.AtEndOfStream Then - l = p.StdErr.ReadLine - If Not merr Is Nothing Then - merr.Write(l) - End If - some = True - End If - WScript.Sleep(100) - Loop +Do Until p.StdOut.AtEndOfStream and p.StdErr.AtEndOfStream and p.Status <> 0 + some = False If Not min Is Nothing Then - min.close() + l = min.ReadLine + p.StdIn.Write(l) + some = True End If - If Not mout Is Nothing Then - mout.close() + If Not p.StdOut.AtEndOfStream Then + l = p.StdOut.ReadLine + If Not mout Is Nothing Then + mout.Write(l) + End If + some = True End If - If Not merr Is Nothing Then - merr.close() + If Not p.StdErr.AtEndOfStream Then + l = p.StdErr.ReadLine + If Not merr Is Nothing Then + merr.Write(l) + End If + some = True End If - If p.ExitCode <> 0 Then - fail "Exit code " + CStr(p.ExitCode), p.ExitCode - End If -Else - fail "Kickstart is not supported on Windows", 250 + WScript.Sleep(100) +Loop +If Not min Is Nothing Then + min.close() End If +If Not mout Is Nothing Then + mout.close() +End If +If Not merr Is Nothing Then + merr.close() +End If +If p.ExitCode <> 0 Then + fail "Exit code " + CStr(p.ExitCode), p.ExitCode +End If shell.CurrentDirectory = WFDIR From hategan at ci.uchicago.edu Fri Jul 4 01:10:07 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:10:07 -0500 (CDT) Subject: [Swift-commit] r7942 - trunk/resources Message-ID: <20140704061007.0B5E99DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:10:06 -0500 (Fri, 04 Jul 2014) New Revision: 7942 Modified: trunk/resources/Karajan.stg Log: removed swift:tr(), swift:stdin(), and other noops Modified: trunk/resources/Karajan.stg =================================================================== --- trunk/resources/Karajan.stg 2014-07-04 02:22:01 UTC (rev 7941) +++ trunk/resources/Karajan.stg 2014-07-04 06:10:06 UTC (rev 7942) @@ -113,8 +113,8 @@ swift_execute(outputs,inputs,stageins,stageouts,attributes,application,name,line) ::= << swift:unitStart("PROCEDURE", name="$name$", line=$line$, outputs="$outputs:list();separator=","$") swift:execute( + "$application.exec$" $attributes$ - swift:tr("$application.exec$") $stageins:swift_stagein();separator="\n"$ $stageouts:swift_stageout();separator="\n"$ $swift_arguments(attributes=application.attributes,arguments=application.arguments, stdin=application.stdin,stdout=application.stdout,stderr=application.stderr)$ @@ -152,15 +152,15 @@ ) $if (stdin)$ -swift:stdin($stdin$) +stdin = $stdin$ $endif$ $if (stdout)$ -swift:stdout($stdout$) +stdout = $stdout$ $endif$ $if (stderr)$ -swift:stderr($stderr$) +stderr = $stderr$ $endif$ >> @@ -309,7 +309,7 @@ swift_mapping(mapping, file) ::= << $if(file)$ -swift:mapping("single_file_mapper", swift:parameter("file", "$file.name$") $if(file.params)$$file.params;separator="\n"$$endif$) +swift:mapping("SingleFileMapper", swift:parameter("file", "$file.name$") $if(file.params)$$file.params;separator="\n"$$endif$) $else$ swift:mapping("$mapping.descriptor$", $mapping.params;separator="\n"$) $endif$ From hategan at ci.uchicago.edu Fri Jul 4 01:11:00 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:11:00 -0500 (CDT) Subject: [Swift-commit] r7943 - trunk/resources Message-ID: <20140704061100.305169DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:10:58 -0500 (Fri, 04 Jul 2014) New Revision: 7943 Modified: trunk/resources/swift-sites-2.0.xsd Log: updated swift sites schema with the new sites spec Modified: trunk/resources/swift-sites-2.0.xsd =================================================================== --- trunk/resources/swift-sites-2.0.xsd 2014-07-04 06:10:06 UTC (rev 7942) +++ trunk/resources/swift-sites-2.0.xsd 2014-07-04 06:10:58 UTC (rev 7943) @@ -5,27 +5,6 @@ targetNamespace="http://www.ci.uchicago.edu/swift/SwiftSites" elementFormDefault="qualified" attributeFormDefault="unqualified" version="1.5"> - - - defines the legal namespaces of execution profiles. - - - - - - - - - - - - - A set of attributes to define a Globus version. - - - - - @@ -49,60 +28,29 @@ - - + - + + - + - root element aggregating all sites information there is. + A name value pair representing a custom property - - - - Describes a single site. - - - - - - - - - - - - - - - - - - - - - - - - Administrative profile defaults associated with a pool. - - - - + @@ -123,7 +71,7 @@ - TODO + A scratch directory. If specified, applications will run in this directory on compute nodes @@ -132,6 +80,9 @@ Describes the mechanism used to submit jobs to the pool + + + @@ -144,97 +95,48 @@ Describes the mechanism used to interact with the filesystem on the given pool + + + - + - Each pool may have multiple gridftp servers. + Allows specifying an environment variable that all applications on this site will see - - - - Stores the bandwidth informaion related to each gridftp server. - - - - - - - - - - - - - - - - The URL (actually, it may be more a URI, but hey, so what) is the access URL to the gridftp server. Each pool may have multiple grid ftp servers, or run multiple versions of Globus on different ports. - - - - - This element is the storage mount point prefix. Of course, this may get turned over into other things, augmented by user and system requirements etc. I believe that default works quite well for default Globus setups. - - - - - + + + + + - + + - Each pool supports various (usually two) jobmanagers. + Specifies the set of applications defined on a site - - - The universe name is actually the primary key for the jobmanager identification. - - - - - The contact string is the secondary key for any job manager. - - - - - Any pool may have multiple versions of Globus installed, and these versions may have multiple jobmanagers listening on different ports. - - - - - - - - - - - - - - + + + + + - + + - Allows specifying an environment variable that all applications on this site will see - - - - - - - Specifies the details of an application installed on a site - + + From hategan at ci.uchicago.edu Fri Jul 4 01:13:23 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:13:23 -0500 (CDT) Subject: [Swift-commit] r7944 - trunk/libexec Message-ID: <20140704061323.7D3A09DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:13:22 -0500 (Fri, 04 Jul 2014) New Revision: 7944 Modified: trunk/libexec/swift-xs.k trunk/libexec/swift.k Log: move type definitions to java (to allow compile-time optimizations) and removed some noops Modified: trunk/libexec/swift-xs.k =================================================================== --- trunk/libexec/swift-xs.k 2014-07-04 06:10:58 UTC (rev 7943) +++ trunk/libexec/swift-xs.k 2014-07-04 06:13:22 UTC (rev 7944) @@ -1,85 +1,10 @@ import(sys) namespace(xs) { - import(java) - - UnresolvedType := function(name, isArray) { - java:new("org.griphyn.vdl.type.impl.UnresolvedType", types=["String"], name) - } - - typesST := function() { - types = ["String", "org.griphyn.vdl.type.Type"] - } - - newSimpleNode := function(name, type) { - x := java:new("org.griphyn.vdl.type.impl.TypeImpl", types=["String"], name) - invokeMethod("setBaseType", object=x, types=["org.griphyn.vdl.type.Type"] - invokeMethod("getType", classname="org.griphyn.vdl.type.Types", type) - ) - x - } - - newComplexNode := function(name) { - java:new("org.griphyn.vdl.type.impl.TypeImpl", types=["String"], name) - } - - addNode := function(node) { - invokeMethod("addType", classname="org.griphyn.vdl.type.Types" - types=["org.griphyn.vdl.type.Type"] - node - ) - } - - addField := function(node, name, type) { - invokeMethod("addField", object=node, typesST(), name, type) - } - - export(schema, - function() { - invokeMethod("resolveTypes", classname="org.griphyn.vdl.type.Types") - } - ) - - export(simpleType, - function(name, type) { - addNode(newSimpleNode(name, type)) - } - ) - - export(restriction, - function(base) { - type = last(split(base, ":")) - } - ) - - export(complexType, - function(name, ...) { - node := newComplexNode(name) - for(field, ...) { - (name, type) := each(field) - addField(node, name, type) - } - addNode(node) - } - ) - - export(sequence, - function(minOccurs = 0, maxOccurs = 0, ...) { - (name, type) := each(first(...)) - if (maxOccurs == "unbounded") { - list(name, UnresolvedType(type, true)) - } - else { - each(...) - } - } - ) - - //should be noted that we're dealing with type names here - export(element, - function(name, type) { - type := last(split(type, ":")) - list(name, UnresolvedType(type, false)) - } - ) + export(schema, def("org.griphyn.vdl.karajan.lib.XS$Schema")) + export(simpleType, def("org.griphyn.vdl.karajan.lib.XS$SimpleType")) + export(restriction, def("org.griphyn.vdl.karajan.lib.XS$Restriction")) + export(complexType, def("org.griphyn.vdl.karajan.lib.XS$ComplexType")) + export(sequence, def("org.griphyn.vdl.karajan.lib.XS$Sequence")) + export(element, def("org.griphyn.vdl.karajan.lib.XS$Element")) } Modified: trunk/libexec/swift.k =================================================================== --- trunk/libexec/swift.k 2014-07-04 06:10:58 UTC (rev 7943) +++ trunk/libexec/swift.k 2014-07-04 06:13:22 UTC (rev 7944) @@ -52,42 +52,10 @@ } ) - export(mapping, - function(descriptor, ...) { - mapping=map(map:entry("swift#descriptor", descriptor), each(...)) - } - ) + export(mapping, def("org.griphyn.vdl.karajan.lib.Mapping$Cons")) + export(parameter, def("org.griphyn.vdl.karajan.lib.Mapping$Parameter")) - export(parameter, - function(name, value) { - map:entry(name, value) - } - ) - export(stdout, - function(file) { - stdout = file - } - ) - - export(stdin, - function(file) { - stdin = file - } - ) - - export(stderr, - function(file) { - stderr = file - } - ) - - export(tr, - function(name) { - tr = name - } - ) - export(attributes, function(attrs) { attributes = attrs From hategan at ci.uchicago.edu Fri Jul 4 01:20:56 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:20:56 -0500 (CDT) Subject: [Swift-commit] r7945 - in trunk/src/org/globus/swift: catalog/site catalog/types data Message-ID: <20140704062056.6C3C99DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:20:56 -0500 (Fri, 04 Jul 2014) New Revision: 7945 Added: trunk/src/org/globus/swift/catalog/site/Application.java trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java Modified: trunk/src/org/globus/swift/catalog/site/SwiftContact.java trunk/src/org/globus/swift/catalog/types/SysInfo.java trunk/src/org/globus/swift/data/Action.java Log: new sites spec Added: trunk/src/org/globus/swift/catalog/site/Application.java =================================================================== --- trunk/src/org/globus/swift/catalog/site/Application.java (rev 0) +++ trunk/src/org/globus/swift/catalog/site/Application.java 2014-07-04 06:20:56 UTC (rev 7945) @@ -0,0 +1,73 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 22, 2014 + */ +package org.globus.swift.catalog.site; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class Application { + private String name, executable; + private Map env; + private Map properties; + + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + + } + public String getExecutable() { + return executable; + } + + public void setExecutable(String executable) { + this.executable = executable; + } + + public void setEnv(String name, String value) { + if (env == null) { + env = new HashMap(); + } + env.put(name, value); + } + + public void addProperty(String name, String value) { + if (properties == null) { + properties = new HashMap(); + } + properties.put(name, value); + } + + public Map getEnv() { + if (env == null) { + return Collections.emptyMap(); + } + else { + return env; + } + } + + public Map getProperties() { + if (properties == null) { + return Collections.emptyMap(); + } + else { + return properties; + } + } + + public boolean executableIsWildcard() { + return "*".equals(executable); + } +} Modified: trunk/src/org/globus/swift/catalog/site/SwiftContact.java =================================================================== --- trunk/src/org/globus/swift/catalog/site/SwiftContact.java 2014-07-04 06:13:22 UTC (rev 7944) +++ trunk/src/org/globus/swift/catalog/site/SwiftContact.java 2014-07-04 06:20:56 UTC (rev 7945) @@ -9,28 +9,72 @@ */ package org.globus.swift.catalog.site; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.globus.cog.karajan.util.BoundContact; -import org.griphyn.vdl.util.FQN; +import org.griphyn.vdl.engine.Warnings; public class SwiftContact extends BoundContact { - private Map profiles = new HashMap(); - + private Map apps; + private SwiftContactSet siteCatalog; + public SwiftContact() { super(); } - public SwiftContact(String host) { - super(host); + public SwiftContact(String name) { + super(name); } - public void addProfile(FQN fqn, String value) { - profiles.put(fqn, value); + public void addApplication(Application app) { + if (apps == null) { + apps = new HashMap(); + } + if (apps.put(app.getName(), app) != null) { + Warnings.warn(Warnings.Type.SITE, "Multiple entries found for application '" + + app.getName() + "' on site '" + this.getName() + "'"); + } } - public Map getProfiles() { - return profiles; + public Collection getApplications() { + if (apps == null) { + return Collections.emptyList(); + } + else { + return apps.values(); + } } + + public SwiftContactSet getSiteCatalog() { + return siteCatalog; + } + + public void setSiteCatalog(SwiftContactSet siteCatalog) { + this.siteCatalog = siteCatalog; + } + + public Application findApplication(String tr) { + /* + * Find apps in the following order: + * host:tr + * host:* + * pool:tr + * pool:* + */ + + Application app = null; + if (apps != null) { + app = apps.get(tr); + if (app == null) { + app = apps.get("*"); + } + } + if (app == null) { + app = siteCatalog.findApplication(tr); + } + return app; + } } Added: trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java =================================================================== --- trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java (rev 0) +++ trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java 2014-07-04 06:20:56 UTC (rev 7945) @@ -0,0 +1,52 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 22, 2014 + */ +package org.globus.swift.catalog.site; + +import java.util.HashMap; +import java.util.Map; + +import org.globus.cog.karajan.util.BoundContact; +import org.globus.cog.karajan.util.ContactSet; +import org.griphyn.vdl.engine.Warnings; + +public class SwiftContactSet extends ContactSet { + private Map apps; + + public void addApplication(Application app) { + if (apps == null) { + apps = new HashMap(); + } + if (apps.put(app.getName(), app) != null) { + Warnings.warn(Warnings.Type.SITE, "Multiple entries found for application '" + + app.getName() + "' on site pool"); + } + } + + @Override + public void addContact(BoundContact contact) { + throw new UnsupportedOperationException(); + } + + public void addContact(SwiftContact contact) { + super.addContact(contact); + contact.setSiteCatalog(this); + } + + public Application findApplication(String tr) { + Application app = null; + if (apps != null) { + app = apps.get(tr); + if (app == null) { + app = apps.get("*"); + } + } + return app; + } +} Modified: trunk/src/org/globus/swift/catalog/types/SysInfo.java =================================================================== --- trunk/src/org/globus/swift/catalog/types/SysInfo.java 2014-07-04 06:13:22 UTC (rev 7944) +++ trunk/src/org/globus/swift/catalog/types/SysInfo.java 2014-07-04 06:20:56 UTC (rev 7945) @@ -227,4 +227,8 @@ } return s.toString(); } + + public static SysInfo fromString(String str) { + return new SysInfo(str); + } } Modified: trunk/src/org/globus/swift/data/Action.java =================================================================== --- trunk/src/org/globus/swift/data/Action.java 2014-07-04 06:13:22 UTC (rev 7944) +++ trunk/src/org/globus/swift/data/Action.java 2014-07-04 06:20:56 UTC (rev 7945) @@ -91,7 +91,7 @@ if (srcdir.length() == 0) { srcdir = "."; } - String desthost = bc.getHost(); + String desthost = bc.getName(); String workdir = (String) bc.getProperty("workdir"); if (workdir != null && !workdir.startsWith("/")) { From hategan at ci.uchicago.edu Fri Jul 4 01:22:05 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:22:05 -0500 (CDT) Subject: [Swift-commit] r7946 - trunk/src/org/griphyn/vdl/engine Message-ID: <20140704062205.C8F2C9DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:22:05 -0500 (Fri, 04 Jul 2014) New Revision: 7946 Modified: trunk/src/org/griphyn/vdl/engine/Warnings.java Log: added a site warning type Modified: trunk/src/org/griphyn/vdl/engine/Warnings.java =================================================================== --- trunk/src/org/griphyn/vdl/engine/Warnings.java 2014-07-04 06:20:56 UTC (rev 7945) +++ trunk/src/org/griphyn/vdl/engine/Warnings.java 2014-07-04 06:22:05 UTC (rev 7946) @@ -22,7 +22,8 @@ public static enum Type { DEPRECATION, SHADOWING, - DATAFLOW + DATAFLOW, + SITE } private static Set warnings = new HashSet(); @@ -31,6 +32,7 @@ static { enabledWarnings.add(Type.DEPRECATION); enabledWarnings.add(Type.DATAFLOW); + enabledWarnings.add(Type.SITE); } public static void warn(Type type, XmlObject obj, String msg) { From hategan at ci.uchicago.edu Fri Jul 4 01:23:08 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:23:08 -0500 (CDT) Subject: [Swift-commit] r7947 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20140704062308.783E29DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:23:06 -0500 (Fri, 04 Jul 2014) New Revision: 7947 Added: trunk/src/org/griphyn/vdl/karajan/lib/XS.java Log: added java version of the type builder Added: trunk/src/org/griphyn/vdl/karajan/lib/XS.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/XS.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/XS.java 2014-07-04 06:23:06 UTC (rev 7947) @@ -0,0 +1,250 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 1, 2014 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.Arrays; +import java.util.List; + +import k.thr.LWThread; + +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.analyzer.Var; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.parser.WrapperNode; +import org.griphyn.vdl.type.DuplicateFieldException; +import org.griphyn.vdl.type.NoSuchTypeException; +import org.griphyn.vdl.type.Type; +import org.griphyn.vdl.type.Types; +import org.griphyn.vdl.type.impl.TypeImpl; +import org.griphyn.vdl.type.impl.UnresolvedType; + +/** + * Type declarations in Swift have no dynamic components. So + * everything can be done at compile time + */ +public class XS { + + public static class Schema extends InternalFunction { + @Override + protected Signature getSignature() { + return new Signature(params()); + } + + @Override + protected void runBody(LWThread thr) { + // do nothing + } + + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { + try { + Types.resolveTypes(); + } + catch (NoSuchTypeException e) { + throw new CompilationException(w, "Cannot resolve types", e); + } + return null; + } + } + + public static class SimpleType extends InternalFunction { + private ArgRef name; + private ArgRef type; + + @Override + protected Signature getSignature() { + return new Signature(params("name", "type")); + } + + @Override + protected void runBody(LWThread thr) { + // do nothing + } + + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { + Type t = new TypeImpl(name.getValue()); + try { + t.setBaseType(Types.getType(type.getValue())); + } + catch (NoSuchTypeException e) { + throw new CompilationException(w, "Unknown type: '" + type.getValue() + "'"); + } + + Types.addType(t); + + return null; + } + } + + public static class Restriction extends InternalFunction { + private ArgRef base; + + @Override + protected Signature getSignature() { + return new Signature(params("base")); + } + + @Override + protected void runBody(LWThread thr) { + // do nothing + } + + /* + * type = last(split(base, ":")) + */ + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { + String base = this.base.getValue(); + + int ix = base.lastIndexOf(':'); + + Var ret = scope.parent.lookupParam("type"); + ret.setValue(base.substring(ix + 1)); + + return null; + } + } + + public static class ComplexType extends InternalFunction { + private ArgRef name; + private ChannelRef c_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("name", "...")); + } + + @Override + protected void runBody(LWThread thr) { + // do nothing + } + + /* + * node := newComplexNode(name) + * for(field, ...) { + * (name, type) := each(field) + * addField(node, name, type) + * } + * addNode(node) + */ + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { + List args = argScope.lookupChannel("...").getAll(); + + Type t = new TypeImpl(name.getValue()); + + for (Object o : args) { + @SuppressWarnings("unchecked") + List l = (List) o; + + String fname = (String) l.get(0); + Type type = (Type) l.get(1); + try { + t.addField(fname, type); + } + catch (DuplicateFieldException e) { + throw new CompilationException(w, "Field '" + fname + "' is alread defined for type '" + name + "'"); + } + } + + Types.addType(t); + + return null; + } + } + + + public static class Sequence extends InternalFunction { + private ArgRef minOccurs; + private ArgRef maxOccurs; + private ChannelRef c_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params(optional("minOccurs", "0"), optional("maxOccurs", "0"), "...")); + } + + @Override + protected void runBody(LWThread thr) { + // do nothing + } + + /* + * (name, type) := each(first(...)) + * if (maxOccurs == "unbounded") { + * list(name, UnresolvedType(type, true)) + * } + * else { + * each(...) + * } + */ + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { + List args = argScope.lookupChannel("...").getAll(); + + Var.Channel crv = scope.parent.lookupChannel("..."); + + if (maxOccurs.getValue().equals("unbounded")) { + @SuppressWarnings("unchecked") + List first = (List) args.get(0); + String name = (String) first.get(0); + String type = (String) first.get(1); + crv.append(Arrays.asList(name, new UnresolvedType(type))); + } + else { + for (Object o : args) { + crv.append(o); + } + } + + return null; + } + } + + + public static class Element extends InternalFunction { + private ArgRef name; + private ArgRef type; + + @Override + protected Signature getSignature() { + return new Signature(params("name", "type")); + } + + @Override + protected void runBody(LWThread thr) { + // do nothing + } + + /* + * type := last(split(type, ":")) + * list(name, UnresolvedType(type, false)) + */ + @Override + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { + String type = this.type.getValue(); + String name = this.name.getValue(); + Var.Channel crv = scope.parent.lookupChannel("..."); + + int ix = type.lastIndexOf(':'); + + crv.append(Arrays.asList(name, new UnresolvedType(type.substring(ix + 1)))); + + return null; + } + } + +} From hategan at ci.uchicago.edu Fri Jul 4 01:24:18 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:24:18 -0500 (CDT) Subject: [Swift-commit] r7948 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20140704062418.DDCD39DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:24:18 -0500 (Fri, 04 Jul 2014) New Revision: 7948 Added: trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java Removed: trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java Log: renamed SiteProfile to SiteProperty Deleted: trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java 2014-07-04 06:23:06 UTC (rev 7947) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java 2014-07-04 06:24:18 UTC (rev 7948) @@ -1,156 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Dec 26, 2006 - */ -package org.griphyn.vdl.karajan.lib; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import k.rt.ExecutionException; -import k.rt.Stack; - -import org.apache.log4j.Logger; -import org.globus.cog.karajan.analyzer.ArgRef; -import org.globus.cog.karajan.analyzer.ChannelRef; -import org.globus.cog.karajan.analyzer.Signature; -import org.globus.cog.karajan.util.BoundContact; -import org.globus.swift.catalog.types.Os; -import org.griphyn.vdl.util.FQN; - -public class SiteProfile extends SwiftFunction { - public static final Logger logger = Logger.getLogger(SiteProfile.class); - - private ArgRef host; - private ArgRef fqn; - private ArgRef _default; - private ChannelRef cr_vargs; - - @Override - protected Signature getSignature() { - return new Signature(params("host", "fqn", optional("default", null)), returns(channel("...", 1))); - } - - public Object function(Stack stack) throws ExecutionException { - BoundContact bc = host.getValue(stack); - return getSingle(bc, new FQN(fqn.getValue(stack)), _default.getValue(stack)); - } - - public static final FQN SWIFT_WRAPPER_INTERPRETER = new FQN("swift:wrapperInterpreter"); - public static final FQN SWIFT_WRAPPER_INTERPRETER_OPTIONS = new FQN("swift:wrapperInterpreterOptions"); - public static final FQN SWIFT_WRAPPER_SCRIPT = new FQN("swift:wrapperScript"); - public static final FQN SWIFT_CLEANUP_COMMAND = new FQN("swift:cleanupCommand"); - public static final FQN SWIFT_CLEANUP_COMMAND_OPTIONS = new FQN("swift:cleanupCommandOptions"); - public static final FQN SYSINFO_OS = new FQN("SYSINFO:OS"); - - private static final Map> DEFAULTS; - private static final Set DEFAULTS_NAMES; - - private static void addDefault(Os os, FQN fqn, Object value) { - DEFAULTS_NAMES.add(fqn); - Map osm = DEFAULTS.get(os); - if (osm == null) { - osm = new HashMap(); - DEFAULTS.put(os, osm); - } - osm.put(fqn, value); - } - - private static boolean hasDefault(Os os, FQN fqn) { - Map osm = DEFAULTS.get(os); - if (osm == null) { - return false; - } - else { - return osm.containsKey(fqn); - } - } - - private static Object getDefault(Os os, FQN fqn) { - Map osm = DEFAULTS.get(os); - if (osm == null) { - osm = DEFAULTS.get(null); - } - return osm.get(fqn); - } - - static { - DEFAULTS = new HashMap>(); - DEFAULTS_NAMES = new HashSet(); - addDefault(Os.WINDOWS, SWIFT_WRAPPER_INTERPRETER, "cscript.exe"); - addDefault(Os.WINDOWS, SWIFT_WRAPPER_SCRIPT, "_swiftwrap.vbs"); - addDefault(Os.WINDOWS, SWIFT_WRAPPER_INTERPRETER_OPTIONS, new String[] {"//Nologo"}); - addDefault(Os.WINDOWS, SWIFT_CLEANUP_COMMAND, "cmd.exe"); - addDefault(Os.WINDOWS, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"/C", "del", "/Q"}); - addDefault(null, SWIFT_WRAPPER_INTERPRETER, "/bin/bash"); - addDefault(null, SWIFT_WRAPPER_SCRIPT, "_swiftwrap"); - addDefault(null, SWIFT_WRAPPER_INTERPRETER_OPTIONS, null); - addDefault(null, SWIFT_CLEANUP_COMMAND, "/bin/rm"); - addDefault(null, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"-rf"}); - } - - private Object getSingle(BoundContact bc, FQN fqn, Object defval) - throws ExecutionException { - String value = getProfile(bc, fqn); - if (value == null) { - Os os = getOS(bc); - if (DEFAULTS_NAMES.contains(fqn)) { - return getDefault(os, fqn); - } - else if (SYSINFO_OS.equals(fqn)) { - return os; - } - else if (defval != null) { - return defval; - } - else { - throw new ExecutionException(this, "Missing profile: " + fqn); - } - } - else { - return value; - } - } - - private String getProfile(BoundContact bc, FQN fqn) { - Object o = bc.getProperty(fqn.toString()); - if (o == null) { - return null; - } - else { - return o.toString(); - } - } - - private Os getOS(BoundContact bc) { - Object o = bc.getProperty("sysinfo"); - if (o == null) { - return Os.LINUX; - } - else { - String[] p = o.toString().split("::"); - if (p.length < 2) { - throw new ExecutionException("Invalid sysinfo for " + bc + ": " + o); - } - return Os.fromString(p[1]); - } - } -} Copied: trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java (from rev 7921, trunk/src/org/griphyn/vdl/karajan/lib/SiteProfile.java) =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java 2014-07-04 06:24:18 UTC (rev 7948) @@ -0,0 +1,152 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* + * Created on Dec 26, 2006 + */ +package org.griphyn.vdl.karajan.lib; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import k.rt.ExecutionException; +import k.rt.Stack; + +import org.apache.log4j.Logger; +import org.globus.cog.karajan.analyzer.ArgRef; +import org.globus.cog.karajan.analyzer.ChannelRef; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.swift.catalog.site.SwiftContact; +import org.globus.swift.catalog.types.Os; +import org.globus.swift.catalog.types.SysInfo; + +public class SiteProperty extends SwiftFunction { + public static final Logger logger = Logger.getLogger(SiteProperty.class); + + private ArgRef host; + private ArgRef name; + private ArgRef _default; + private ChannelRef cr_vargs; + + @Override + protected Signature getSignature() { + return new Signature(params("host", "name", optional("default", null)), returns(channel("...", 1))); + } + + public Object function(Stack stack) throws ExecutionException { + SwiftContact bc = host.getValue(stack); + return getSingle(bc, name.getValue(stack), _default.getValue(stack)); + } + + public static final String SWIFT_WRAPPER_INTERPRETER = "wrapperInterpreter"; + public static final String SWIFT_WRAPPER_INTERPRETER_OPTIONS = "wrapperInterpreterOptions"; + public static final String SWIFT_WRAPPER_SCRIPT = "wrapperScript"; + public static final String SWIFT_CLEANUP_COMMAND = "cleanupCommand"; + public static final String SWIFT_CLEANUP_COMMAND_OPTIONS = "cleanupCommandOptions"; + public static final String SYSINFO_OS = "OS"; + + private static final Map> DEFAULTS; + private static final Set DEFAULTS_NAMES; + + private static void addDefault(Os os, String name, Object value) { + DEFAULTS_NAMES.add(name); + Map osm = DEFAULTS.get(os); + if (osm == null) { + osm = new HashMap(); + DEFAULTS.put(os, osm); + } + osm.put(name, value); + } + + private static boolean hasDefault(Os os, String name) { + Map osm = DEFAULTS.get(os); + if (osm == null) { + return false; + } + else { + return osm.containsKey(name); + } + } + + private static Object getDefault(Os os, String name) { + Map osm = DEFAULTS.get(os); + if (osm == null) { + osm = DEFAULTS.get(null); + } + return osm.get(name); + } + + static { + DEFAULTS = new HashMap>(); + DEFAULTS_NAMES = new HashSet(); + addDefault(Os.WINDOWS, SWIFT_WRAPPER_INTERPRETER, "cscript.exe"); + addDefault(Os.WINDOWS, SWIFT_WRAPPER_SCRIPT, "_swiftwrap.vbs"); + addDefault(Os.WINDOWS, SWIFT_WRAPPER_INTERPRETER_OPTIONS, new String[] {"//Nologo"}); + addDefault(Os.WINDOWS, SWIFT_CLEANUP_COMMAND, "cmd.exe"); + addDefault(Os.WINDOWS, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"/C", "del", "/Q"}); + addDefault(null, SWIFT_WRAPPER_INTERPRETER, "/bin/bash"); + addDefault(null, SWIFT_WRAPPER_SCRIPT, "_swiftwrap"); + addDefault(null, SWIFT_WRAPPER_INTERPRETER_OPTIONS, null); + addDefault(null, SWIFT_CLEANUP_COMMAND, "/bin/rm"); + addDefault(null, SWIFT_CLEANUP_COMMAND_OPTIONS, new String[] {"-rf"}); + } + + private Object getSingle(SwiftContact bc, String name, Object defval) + throws ExecutionException { + String value = getProperty(bc, name); + if (value == null) { + Os os = getOS(bc); + if (DEFAULTS_NAMES.contains(name)) { + return getDefault(os, name); + } + else if (SYSINFO_OS.equals(name)) { + return os; + } + else if (defval != null) { + return defval; + } + else { + throw new ExecutionException(this, "Missing profile: " + name); + } + } + else { + return value; + } + } + + private String getProperty(SwiftContact bc, String name) { + Object o = bc.getProperty(name); + if (o == null) { + return null; + } + else { + return o.toString(); + } + } + + private Os getOS(SwiftContact bc) { + Object o = bc.getProperty("sysinfo"); + if (o == null) { + return Os.LINUX; + } + else { + return SysInfo.fromString(o.toString()).getOs(); + } + } +} From hategan at ci.uchicago.edu Fri Jul 4 01:24:42 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:24:42 -0500 (CDT) Subject: [Swift-commit] r7949 - trunk/libexec Message-ID: <20140704062442.C0C969DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:24:42 -0500 (Fri, 04 Jul 2014) New Revision: 7949 Modified: trunk/libexec/swift-lib.k Log: renamed SiteProfile to SiteProperty Modified: trunk/libexec/swift-lib.k =================================================================== --- trunk/libexec/swift-lib.k 2014-07-04 06:24:18 UTC (rev 7948) +++ trunk/libexec/swift-lib.k 2014-07-04 06:24:42 UTC (rev 7949) @@ -83,7 +83,7 @@ export(executable, def("org.griphyn.vdl.karajan.lib.Executable")) export(TCProfile, def("org.griphyn.vdl.karajan.lib.TCProfile")) - export(siteProfile, def("org.griphyn.vdl.karajan.lib.SiteProfile")) + export(siteProfile, def("org.griphyn.vdl.karajan.lib.SiteProperty")) export(setFutureFault, def("org.griphyn.vdl.karajan.lib.SetFutureFault")) From hategan at ci.uchicago.edu Fri Jul 4 01:32:22 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:32:22 -0500 (CDT) Subject: [Swift-commit] r7950 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20140704063222.E53DE9DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:32:22 -0500 (Fri, 04 Jul 2014) New Revision: 7950 Modified: trunk/src/org/griphyn/vdl/karajan/lib/Executable.java trunk/src/org/griphyn/vdl/karajan/lib/Execute.java trunk/src/org/griphyn/vdl/karajan/lib/JobConstraints.java trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java Log: new sites spec Modified: trunk/src/org/griphyn/vdl/karajan/lib/Executable.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Executable.java 2014-07-04 06:24:42 UTC (rev 7949) +++ trunk/src/org/griphyn/vdl/karajan/lib/Executable.java 2014-07-04 06:32:22 UTC (rev 7950) @@ -24,15 +24,13 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Signature; -import org.globus.cog.karajan.util.BoundContact; -import org.globus.swift.catalog.TCEntry; -import org.griphyn.vdl.karajan.TCCache; -import org.griphyn.vdl.util.FQN; +import org.globus.swift.catalog.site.Application; +import org.globus.swift.catalog.site.SwiftContact; public class Executable extends SwiftFunction { private ArgRef tr; - private ArgRef host; + private ArgRef host; @Override @@ -42,21 +40,16 @@ public Object function(Stack stack) { - TCCache tc = getTC(stack); String tr = this.tr.getValue(stack); - BoundContact bc = this.host.getValue(stack); - TCEntry tce = getTCE(tc, new FQN(tr), bc); - if (tce == null) { + SwiftContact bc = this.host.getValue(stack); + // at this point, a host has been allocated, so we already + // know that the app is available on it + Application app = bc.findApplication(tr); + if (app.executableIsWildcard()) { return tr; } else { - String pt = tce.getPhysicalTransformation(); - if ("*".equals(pt)) { - return tr; - } - else { - return pt; - } + return app.getExecutable(); } } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2014-07-04 06:24:42 UTC (rev 7949) +++ trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2014-07-04 06:32:22 UTC (rev 7950) @@ -24,6 +24,7 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import k.rt.Abort; @@ -35,6 +36,8 @@ import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.StatusEvent; +import org.globus.cog.abstraction.impl.common.task.JobSpecificationImpl; +import org.globus.cog.abstraction.interfaces.JobSpecification; import org.globus.cog.abstraction.interfaces.Status; import org.globus.cog.abstraction.interfaces.Task; import org.globus.cog.karajan.analyzer.ArgRef; @@ -58,6 +61,7 @@ private ArgRef> replicationChannel; private ArgRef jobid; private ArgRef progress; + private ArgRef> environment; private VarRef context; @@ -73,7 +77,8 @@ params.add(optional("jobid", null)); removeParams(params, "stdout", "stderr", "stdoutLocation", "stderrLocation", "stdin", "provider", "securityContext", "nativespec", - "delegation", "batch"); + "delegation", "batch", "environment"); + params.add(optional("environment", null)); return sig; } @@ -103,6 +108,11 @@ } @Override + protected void addEnvironment(Stack stack, JobSpecificationImpl js) throws ExecutionException { + js.setEnvironmentVariables(environment.getValue(stack)); + } + + @Override public void submitScheduled(Scheduler scheduler, Task task, Stack stack, Object constraints) { try { setTaskIdentity(stack, task); @@ -129,12 +139,28 @@ logger.debug("Submitting task " + task); } String jobid = this.jobid.getValue(stack); - if (logger.isDebugEnabled()) { - logger.debug("jobid=" + jobid + " task=" + task); + if (logger.isInfoEnabled()) { + JobSpecification spec = (JobSpecification) task.getSpecification(); + logger.info(buildTaskInfoString(task, spec)); } } - protected void registerReplica(Stack stack, Task task) throws CanceledReplicaException { + private String buildTaskInfoString(Task task, JobSpecification spec) { + StringBuilder sb = new StringBuilder(); + sb.append("JOB_TASK jobid="); + sb.append(jobid); + sb.append(" taskid="); + sb.append(task.getIdentity()); + sb.append(" exec="); + sb.append(spec.getExecutable()); + sb.append(" dir="); + sb.append(spec.getDirectory()); + sb.append(" args="); + sb.append(spec.getArguments()); + return sb.toString(); + } + + protected void registerReplica(Stack stack, Task task) throws CanceledReplicaException { String rg = this.replicationGroup.getValue(stack); if (rg != null) { getReplicationManager(stack).register(rg, task); @@ -152,12 +178,22 @@ Stack stack = getStack(); try { if (stack != null) { - int c = e.getStatus().getStatusCode(); + Status s = e.getStatus(); + int c = s.getStatusCode(); + if (logger.isInfoEnabled()) { + if (s.getMessage() == null) { + logger.info("TASK_STATUS_CHANGE taskid=" + e.getSource().getIdentity() + " status=" + c); + } + else { + logger.info("TASK_STATUS_CHANGE taskid=" + e.getSource().getIdentity() + " status=" + c + + " " + s.getMessage()); + } + } ProgressState ps = progress.getValue(stack); if (c == Status.SUBMITTED) { ps.setState("Submitted"); if (replicationEnabled) { - getReplicationManager(stack).submitted(task, e.getStatus().getTime()); + getReplicationManager(stack).submitted(task, s.getTime()); } } else if (c == Status.STAGE_IN) { @@ -169,7 +205,7 @@ else if (c == Status.ACTIVE) { ps.setState("Active"); if (replicationEnabled) { - getReplicationManager(stack).active(task, e.getStatus().getTime()); + getReplicationManager(stack).active(task, s.getTime()); Execute.this.replicationChannel.getValue(stack).close(); } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/JobConstraints.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/JobConstraints.java 2014-07-04 06:24:42 UTC (rev 7949) +++ trunk/src/org/griphyn/vdl/karajan/lib/JobConstraints.java 2014-07-04 06:32:22 UTC (rev 7950) @@ -33,7 +33,6 @@ import org.globus.cog.karajan.scheduler.TaskConstraints; import org.griphyn.vdl.karajan.lib.cache.CacheMapAdapter; import org.griphyn.vdl.mapping.DSHandle; -import org.griphyn.vdl.util.FQN; public class JobConstraints extends CacheFunction { private ArgRef tr; @@ -53,7 +52,7 @@ String tr = this.tr.getValue(stack); String[] filenames = null; Collection stageins = this.stagein.getValue(stack); - SwiftTaskConstraints tc = new SwiftTaskConstraints(tr, new FQN(tr)); + SwiftTaskConstraints tc = new SwiftTaskConstraints(tr); if (stageins != null) { tc.setStageins(stageins); tc.setFilecache(new CacheMapAdapter(getCache(stack))); @@ -67,13 +66,11 @@ private static class SwiftTaskConstraints implements TaskConstraints { private final String tr; - private final FQN trfqn; private Collection stageins; private CacheMapAdapter filecache; - public SwiftTaskConstraints(String tr, FQN trfqn) { + public SwiftTaskConstraints(String tr) { this.tr = tr; - this.trfqn = trfqn; } public Collection getStageins() { @@ -98,9 +95,6 @@ if ("tr".equals(name)) { return tr; } - else if ("trfqn".equals(name)) { - return trfqn; - } else if ("stageins".equals(name)) { return stageins; } Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-07-04 06:24:42 UTC (rev 7949) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-07-04 06:32:22 UTC (rev 7950) @@ -9,6 +9,9 @@ */ package org.griphyn.vdl.karajan.lib; +import java.util.ArrayList; +import java.util.List; + import k.rt.ExecutionException; import k.rt.Stack; @@ -24,11 +27,10 @@ import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Param; import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; -import org.globus.cog.karajan.util.BoundContact; -import org.globus.cog.karajan.util.ContactSet; +import org.globus.swift.catalog.site.Application; import org.globus.swift.catalog.site.SiteCatalogParser; import org.globus.swift.catalog.site.SwiftContact; -import org.griphyn.vdl.util.FQN; +import org.globus.swift.catalog.site.SwiftContactSet; import org.w3c.dom.Document; import org.w3c.dom.NamedNodeMap; import org.w3c.dom.Node; @@ -36,10 +38,6 @@ public class SiteCatalog extends AbstractSingleValuedFunction { private ArgRef fileName; - - private enum Version { - V1, V2; - } @Override protected Param[] getParams() { @@ -58,48 +56,59 @@ throw new ExecutionException(this, "Failed to parse site catalog", e); } } + + private static class KVPair { + public final String key; + public final String value; + + public KVPair(String key, String value) { + this.key = key; + this.value = value; + } + } private Object buildResources(Document doc) { Node root = getRoot(doc); if (root.getLocalName().equals("config")) { - return parse(root, Version.V1); + throw new IllegalArgumentException("Old sites file format. Please upgrade your sites file to the new format."); } else if (root.getLocalName().equals("sites")) { - return parse(root, Version.V1); + return parse(root); } else { throw new IllegalArgumentException("Illegal sites file root node: " + root.getLocalName()); } } - - - - private Object parse(Node config, Version v) { - ContactSet cs = new ContactSet(); + + private Object parse(Node config) { + SwiftContactSet cs = new SwiftContactSet(); NodeList pools = config.getChildNodes(); for (int i = 0; i < pools.getLength(); i++) { Node n = pools.item(i); if (n.getNodeType() == Node.ELEMENT_NODE) { - try { - BoundContact bc = pool(n, v); - if (bc != null) { - cs.addContact(bc); + String ctype = n.getNodeName(); + if (ctype.equals("site")) { + try { + SwiftContact bc = pool(n); + if (bc != null) { + cs.addContact(bc); + } } + catch (Exception e) { + throw new ExecutionException(this, "Invalid site entry '" + poolName(n) + "': ", e); + } } - catch (Exception e) { - throw new ExecutionException(this, "Invalid site entry '" + poolName(n, v) + "': ", e); + else if (ctype.equals("application")) { + cs.addApplication(application(n)); } } } return cs; } - private String poolName(Node site, Version v) { - if (site.getLocalName().equals("pool")) { - return attr(site, "handle"); - } - else if (site.getLocalName().equals("site")) { + private String poolName(Node site) { + if (site.getLocalName().equals("site")) { return attr(site, "name"); } else { @@ -117,18 +126,18 @@ throw new IllegalArgumentException("Missing root element"); } - private BoundContact pool(Node n, Version v) throws InvalidProviderException, ProviderMethodException { + private SwiftContact pool(Node n) throws InvalidProviderException, ProviderMethodException { if (n.getNodeType() != Node.ELEMENT_NODE) { return null; } - String name = poolName(n, v); + String name = poolName(n); SwiftContact bc = new SwiftContact(name); String sysinfo = attr(n, "sysinfo", null); if (sysinfo != null) { - bc.addProperty("sysinfo", sysinfo); + bc.setProperty("sysinfo", sysinfo); } - + NodeList cs = n.getChildNodes(); for (int i = 0; i < cs.getLength(); i++) { @@ -138,33 +147,24 @@ } String ctype = c.getNodeName(); - if (v == Version.V1 && ctype.equals("gridftp")) { - bc.addService(gridftp(c)); - } - else if (v == Version.V1 && ctype.equals("jobmanager")) { - bc.addService(jobmanager(c)); - } - else if (ctype.equals("execution")) { + if (ctype.equals("execution")) { bc.addService(execution(c)); } else if (ctype.equals("filesystem")) { bc.addService(filesystem(c)); } else if (ctype.equals("workdirectory")) { - bc.addProperty("workdir", text(c)); + bc.setProperty("workdir", text(c)); } else if (ctype.equals("scratch")) { - bc.addProperty("scratch", text(c)); + bc.setProperty("scratch", text(c)); } - else if (ctype.equals("env")) { - env(bc, c); + else if (ctype.equals("property")) { + bc.setProperty(attr(c, "name"), text(c)); } - else if (ctype.equals("profile")) { - profile(bc, c); + else if (ctype.equals("apps")) { + apps(bc, c); } - else if (v == Version.V2 && ctype.equals("application")) { - application(bc, c); - } else { throw new IllegalArgumentException("Unknown node type: " + ctype); } @@ -172,44 +172,85 @@ return bc; } - private void application(BoundContact bc, Node c) { + private void apps(SwiftContact bc, Node n) { + NodeList cs = n.getChildNodes(); + + List envs = new ArrayList(); + List props = new ArrayList(); + for (int i = 0; i < cs.getLength(); i++) { + Node c = cs.item(i); + if (c.getNodeType() != Node.ELEMENT_NODE) { + continue; + } + String ctype = c.getNodeName(); + + if (ctype.equals("app")) { + bc.addApplication(application(c)); + } + else if (ctype.equals("env")) { + envs.add(env(c)); + } + else if (ctype.equals("property")) { + props.add(this.property(c)); + } + else { + throw new IllegalArgumentException("Unknown node type: " + ctype); + } + } + + mergeEnvsToApps(bc, envs); + mergePropsToApps(bc, props); } - private Service jobmanager(Node n) throws InvalidProviderException, ProviderMethodException { - String provider; - String url = attr(n, "url"); - String major = attr(n, "major"); - if (url.equals("local://localhost")) { - provider = "local"; + private void mergeEnvsToApps(SwiftContact bc, List envs) { + for (Application app : bc.getApplications()) { + for (KVPair kvp : envs) { + if (!app.getEnv().containsKey(kvp.key)) { + // only merge if app does not override + app.setEnv(kvp.key, kvp.value); + } + } } - else if (url.equals("pbs://localhost")) { - provider = "pbs"; + } + + private void mergePropsToApps(SwiftContact bc, List props) { + for (Application app : bc.getApplications()) { + for (KVPair kvp : props) { + if (!app.getProperties().containsKey(kvp.key)) { + app.addProperty(kvp.key, kvp.value); + } + } } - else if ("2".equals(major)) { - provider = "gt2"; - } - else if ("4".equals(major)) { - provider = "gt4"; - } - else { - throw new IllegalArgumentException("Unknown job manager version: " + major + ", url = '" + url + "'"); - } - - ServiceContact contact = new ServiceContactImpl(url); - return new ServiceImpl(provider, Service.EXECUTION, - contact, AbstractionFactory.newSecurityContext(provider, contact)); } - private Service gridftp(Node n) throws InvalidProviderException, ProviderMethodException { - String url = attr(n, "url"); - if (url.equals("local://localhost")) { - return new ServiceImpl("local", Service.FILE_OPERATION, new ServiceContactImpl("localhost"), null); + private Application application(Node n) { + Application app = new Application(); + app.setName(attr(n, "name")); + app.setExecutable(attr(n, "executable")); + + NodeList cs = n.getChildNodes(); + + for (int i = 0; i < cs.getLength(); i++) { + Node c = cs.item(i); + if (c.getNodeType() != Node.ELEMENT_NODE) { + continue; + } + String ctype = c.getNodeName(); + + if (ctype.equals("env")) { + KVPair env = env(c); + app.setEnv(env.key, env.value); + } + else if (ctype.equals("property")) { + KVPair prop = property(c); + app.addProperty(prop.key, prop.value); + } + else { + throw new IllegalArgumentException("Unknown node type: " + ctype); + } } - else { - ServiceContact contact = new ServiceContactImpl(url); - return new ServiceImpl("gsiftp", Service.FILE_OPERATION, - contact, AbstractionFactory.newSecurityContext("gsiftp", contact)); - } + + return app; } private Service execution(Node n) throws InvalidProviderException, ProviderMethodException { @@ -233,9 +274,39 @@ s.setJobManager(jobManager); } + properties(s, n); + return s; } + private void properties(Service s, Node n) { + NodeList cs = n.getChildNodes(); + + for (int i = 0; i < cs.getLength(); i++) { + Node c = cs.item(i); + if (c.getNodeType() != Node.ELEMENT_NODE) { + continue; + } + String ctype = c.getNodeName(); + + if (ctype.equals("property")) { + property(s, c); + } + else { + throw new IllegalArgumentException("Unknown node type: " + ctype); + } + } + + } + + private void property(Service s, Node c) { + s.setAttribute(attr(c, "name"), text(c)); + } + + private KVPair property(Node c) { + return new KVPair(attr(c, "name"), text(c)); + } + private Service filesystem(Node n) throws InvalidProviderException, ProviderMethodException { String provider = attr(n, "provider"); String url = attr(n, "url", null); @@ -251,32 +322,18 @@ s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact)); } + properties(s, n); + return s; } - - private void env(SwiftContact bc, Node n) { - String key = attr(n, "key"); + + private KVPair env(Node n) { + String key = attr(n, "name"); String value = text(n); - bc.addProfile(new FQN("env", key), value); + return new KVPair(key, value); } - private void profile(SwiftContact bc, Node n) { - String ns = attr(n, "namespace"); - String key = attr(n, "key"); - String value = text(n); - - if (value == null) { - throw new IllegalArgumentException("No value for profile " + ns + ":" + key); - } - if (ns.equals("karajan")) { - bc.addProperty(key, value); - } - else { - bc.addProfile(new FQN(ns, key), value); - } - } - private String text(Node n) { if (n.getFirstChild() != null) { return n.getFirstChild().getNodeValue(); Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2014-07-04 06:24:42 UTC (rev 7949) +++ trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2014-07-04 06:32:22 UTC (rev 7950) @@ -21,11 +21,8 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import k.rt.Channel; import k.rt.Context; @@ -41,14 +38,8 @@ import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.compiled.nodes.functions.AbstractFunction; import org.globus.cog.karajan.parser.WrapperNode; -import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.swift.catalog.TCEntry; -import org.globus.swift.catalog.transformation.File; -import org.globus.swift.catalog.types.TCType; import org.griphyn.vdl.karajan.AssertFailedException; -import org.griphyn.vdl.karajan.TCCache; -import org.griphyn.vdl.karajan.functions.ConfigProperty; import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DependentException; @@ -65,9 +56,7 @@ import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; -import org.griphyn.vdl.util.FQN; import org.griphyn.vdl.util.VDL2Config; -import org.griphyn.vdl.util.VDL2ConfigProperties; public abstract class SwiftFunction extends AbstractFunction { public static final Logger logger = Logger.getLogger(SwiftFunction.class); @@ -386,50 +375,7 @@ return Path.parse((String) o); } } - - private static Set> warnset = new HashSet>(); - - protected TCEntry getTCE(TCCache tc, FQN fqn, BoundContact bc) { - List l; - try { - l = tc.getTCEntries(fqn, bc.getHost(), TCType.INSTALLED); - } - catch (Exception e) { - throw new ExecutionException(this, e); - } - if (l == null || l.isEmpty()) { - return null; - } - if (l.size() > 1) { - synchronized (warnset) { - LinkedList wl = new LinkedList(); - wl.add(fqn); - wl.add(bc); - if (!warnset.contains(wl)) { - logger.warn("Multiple entries found for " + fqn + " on " + bc - + ". Using the first one"); - warnset.add(wl); - } - } - } - return l.get(0); - } - - public static final String TC = "vdl:TC"; - - public TCCache getTC(Stack stack) throws ExecutionException { - Context c = this.context.getValue(stack); - synchronized (c) { - TCCache tc = (TCCache) c.getAttribute(TC); - if (tc == null) { - String prop = ConfigProperty.getProperty(VDL2ConfigProperties.TC_FILE, (VDL2Config) c.getAttribute("SWIFT:CONFIG")); - tc = new TCCache(File.getNonSingletonInstance(prop)); - c.setAttribute(TC, tc); - } - return tc; - } - } - + private static int provenanceIDCount = 451000; public static synchronized int nextProvenanceID() { Modified: trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2014-07-04 06:24:42 UTC (rev 7949) +++ trunk/src/org/griphyn/vdl/karajan/lib/TCProfile.java 2014-07-04 06:32:22 UTC (rev 7950) @@ -21,11 +21,7 @@ package org.griphyn.vdl.karajan.lib; import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; import java.util.Map; -import java.util.Set; import k.rt.ExecutionException; import k.rt.Stack; @@ -33,15 +29,11 @@ import org.apache.log4j.Logger; import org.globus.cog.abstraction.impl.common.execution.WallTime; import org.globus.cog.karajan.analyzer.ArgRef; -import org.globus.cog.karajan.analyzer.ChannelRef; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.VarRef; -import org.globus.cog.karajan.compiled.nodes.functions.Map.Entry; -import org.globus.swift.catalog.TCEntry; +import org.globus.swift.catalog.site.Application; import org.globus.swift.catalog.site.SwiftContact; -import org.globus.swift.catalog.util.Profile; -import org.griphyn.vdl.karajan.TCCache; -import org.griphyn.vdl.util.FQN; +import org.griphyn.vdl.engine.Warnings; public class TCProfile extends SwiftFunction { public static final Logger logger = Logger.getLogger(TCProfile.class); @@ -58,7 +50,7 @@ private VarRef r_count; private VarRef r_jobType; private VarRef r_attributes; - private ChannelRef> cr_environment; + private VarRef> r_environment; private enum Attr { COUNT, JOB_TYPE; @@ -77,66 +69,67 @@ return new Signature( params("host", optional("attributes", null), optional("tr", null)), returns("count", "jobType", - "attributes", channel("environment", DYNAMIC)) + "attributes", "environment") ); } public Object function(Stack stack) { - TCCache tc = getTC(stack); String tr = this.tr.getValue(stack); - Map dynamicAttributes = readDynamicAttributes(stack); - SwiftContact bc = this.host.getValue(stack); - - Map attrs = null; - attrs = attributesFromHost(bc, attrs, stack); - - TCEntry tce = null; - if (tr != null) { - tce = getTCE(tc, new FQN(tr), bc); + Application app = bc.findApplication(tr); + if (app == null) { + throw new RuntimeException("Application '" + tr + "' not found on site '" + bc.getName() + "'"); } - if (tce != null) { - addEnvironment(stack, tce); - addEnvironment(stack, bc); - attrs = attributesFromTC(tce, attrs, stack); - } - attrs = addDynamicAttributes(attrs, dynamicAttributes); - checkWalltime(attrs, tr, stack); - addAttributes(attrs, stack); + addEnvironment(stack, app.getEnv()); + + Map dynamicAttributes = this.attributes.getValue(stack); + Map attrs = null; + + attrs = combineAttributes(stack, attrs, app.getProperties()); + attrs = combineAttributes(stack, attrs, dynamicAttributes); + + checkWalltime(stack, attrs, tr); + setAttributes(stack, attrs); return null; } - /** + private void addEnvironment(Stack stack, Map env) { + r_environment.setValue(stack, env); + } + + /** Bring in the dynamic attributes from the Karajan stack @return Map, may be null */ private Map readDynamicAttributes(Stack stack) { return this.attributes.getValue(stack); } - + /** - Store dynamic attributes into returned attributes, - overwriting if necessary - @param result Attributes so far known, may be null - @param dynamicAttributes Attributes to insert, may be null - @result Combination, may be null + * Combine attributes creating result as necessary */ - private Map - addDynamicAttributes(Map result, - Map dynamicAttributes) { - if (result == null && dynamicAttributes == null) - return null; - if (result == null) - return dynamicAttributes; - if (dynamicAttributes == null) - return result; - result.putAll(dynamicAttributes); + private Map combineAttributes(Stack stack, Map result, Map src) { + if (src == null || src.isEmpty()) { + return result; + } + for (Map.Entry e : src.entrySet()) { + Attr a = ATTR_TYPES.get(e.getKey()); + if (a != null) { + setAttr(a, stack, e.getValue()); + } + else { + if (result == null) { + result = new HashMap(); + } + result.put(e.getKey(), e.getValue()); + } + } return result; } - private void checkWalltime(Map attrs, String tr, Stack stack) { + private void checkWalltime(Stack stack, Map attrs, String tr) { if (attrs == null) { return; } @@ -149,113 +142,15 @@ WallTime.timeToSeconds(walltime.toString()); } catch (ExecutionException e) { - warn(tr, "Warning: invalid walltime specification for \"" + tr + Warnings.warn(Warnings.Type.SITE, "Invalid walltime specification for \"" + tr + "\" (" + walltime + ")."); } } - - private static final Set warnedAboutWalltime = - new HashSet(); - - private void warn(String tr, String message) { - synchronized (warnedAboutWalltime) { - if (warnedAboutWalltime.add(tr)) { - System.out.println(message); - } - } - } - - private void addEnvironment(Stack stack, TCEntry tce) { - List list = tce.getProfiles(Profile.ENV); - if (list != null) { - for (Profile p : list) { - cr_environment.append(stack, new Entry(p.getProfileKey(), p.getProfileValue())); - } - } - } - - public static final String PROFILE_GLOBUS_PREFIX = (Profile.GLOBUS + "::").toLowerCase(); - - private void addEnvironment(Stack stack, SwiftContact bc) { - Map profiles = bc.getProfiles(); - if (profiles != null) { - for (Map.Entry e : profiles.entrySet()) { - FQN fqn = e.getKey(); - String value = (String) e.getValue(); - if (Profile.ENV.equalsIgnoreCase(fqn.getNamespace())) { - cr_environment.append(stack, new Entry(fqn.getName(), value)); - } - } - } - } - - private void addAttributes(Map attrs, Stack stack) { - if (logger.isDebugEnabled()) { - logger.debug("Attributes: " + attrs); - } - if (attrs == null) { - return; - } - Iterator> i = attrs.entrySet().iterator(); - while (i.hasNext()) { - Map.Entry e = i.next(); - Attr a = ATTR_TYPES.get(e.getKey()); - if (a != null) { - setAttr(a, stack, e.getValue()); - i.remove(); - } - } - if (attrs.size() == 0) { - return; - } + + private void setAttributes(Stack stack, Map attrs) { this.r_attributes.setValue(stack, attrs); } - private Map attributesFromTC(TCEntry tce, Map attrs, Stack stack) { - List list = tce.getProfiles(Profile.GLOBUS); - if (list != null) { - for (Profile p : list) { - Attr a = ATTR_TYPES.get(p.getProfileKey()); - if (a == null) { - if (attrs == null) { - attrs = new HashMap(); - } - attrs.put(p.getProfileKey(), p.getProfileValue()); - } - else { - setAttr(a, stack, p.getProfileValue()); - } - } - } - return attrs; - } - - /** - Inserts namespace=globus attributes from BoundContact bc - into given attrs - */ - private Map attributesFromHost(SwiftContact bc, Map attrs, Stack stack) { - Map profiles = bc.getProfiles(); - if (profiles != null) { - for (Map.Entry e : profiles.entrySet()) { - FQN fqn = e.getKey(); - if (Profile.GLOBUS.equalsIgnoreCase(fqn.getNamespace())) { - Attr a = ATTR_TYPES.get(fqn.getName()); - if (a == null) { - if (attrs == null) { - attrs = new HashMap(); - } - attrs.put(fqn.getName(), e.getValue()); - } - else { - setAttr(a, stack, e.getValue()); - } - } - } - } - return attrs; - } - private void setAttr(Attr a, Stack stack, Object value) { switch (a) { case COUNT: From hategan at ci.uchicago.edu Fri Jul 4 01:33:48 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:33:48 -0500 (CDT) Subject: [Swift-commit] r7951 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20140704063348.DB9339DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:33:48 -0500 (Fri, 04 Jul 2014) New Revision: 7951 Removed: trunk/src/org/griphyn/vdl/karajan/TCCache.java Log: removed tc cache since tc.data is not used any more Deleted: trunk/src/org/griphyn/vdl/karajan/TCCache.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/TCCache.java 2014-07-04 06:32:22 UTC (rev 7950) +++ trunk/src/org/griphyn/vdl/karajan/TCCache.java 2014-07-04 06:33:48 UTC (rev 7951) @@ -1,119 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Jan 5, 2007 - */ -package org.griphyn.vdl.karajan; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.globus.swift.catalog.TCEntry; -import org.globus.swift.catalog.TransformationCatalog; -import org.globus.swift.catalog.types.TCType; -import org.griphyn.vdl.util.FQN; - -public class TCCache { - private TransformationCatalog tc; - private Map> cache; - private Entry entry; - - public TCCache(TransformationCatalog tc) { - this.tc = tc; - cache = new HashMap>(); - entry = new Entry(); - } - - public static final FQN ANY_APP = new FQN("*"); - - public synchronized List getTCEntries(FQN tr, String host, TCType tctype) throws Exception { - List l; - // try exact entry - entry.set(tr, host, tctype); - l = getTCEntries_(entry); - if (l != null) { - return l; - } - - // try app wildcard on this host - entry.set(ANY_APP, host, tctype); - l = getTCEntries_(entry); - if (l != null) { - return l; - } - // try this app on wildcard host - entry.set(tr, "*", tctype); - l = getTCEntries_(entry); - if (l != null) { - return l; - } - - // finally try app wildcard on wildcard host - entry.set(ANY_APP, "*", tctype); - l = getTCEntries_(entry); - return l; - } - - private List getTCEntries_(Entry e) throws Exception { - List l = cache.get(e); - if (l == null && !cache.containsKey(e)) { - l = tc.getTCEntries(e.tr.getNamespace(), e.tr.getName(), e.tr.getVersion(), e.host, e.tctype); - cache.put(new Entry(e), l); - } - - return l; - } - - private class Entry { - public FQN tr; - public String host; - public TCType tctype; - - public Entry() { - } - - public Entry(FQN tr, String host, TCType tctype) { - set(tr, host, tctype); - } - - public Entry(Entry e) { - set(e.tr, e.host, e.tctype); - } - - public void set(FQN tr, String host, TCType tctype) { - this.tr = tr; - this.host = host; - this.tctype = tctype; - } - - public boolean equals(Object obj) { - if (obj instanceof Entry) { - Entry other = (Entry) obj; - return tr.equals(other.tr) && host.equals(other.host) && tctype.equals(other.tctype); - } - else { - return false; - } - } - - public int hashCode() { - return tr.hashCode() + host.hashCode() + tctype.hashCode(); - } - } -} From hategan at ci.uchicago.edu Fri Jul 4 01:34:39 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:34:39 -0500 (CDT) Subject: [Swift-commit] r7952 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20140704063439.4BEB19DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:34:38 -0500 (Fri, 04 Jul 2014) New Revision: 7952 Modified: trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java Log: new sites spec Modified: trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2014-07-04 06:33:48 UTC (rev 7951) +++ trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2014-07-04 06:34:38 UTC (rev 7952) @@ -44,16 +44,14 @@ import org.globus.cog.karajan.scheduler.ContactAllocationTask; import org.globus.cog.karajan.scheduler.ResourceConstraintChecker; import org.globus.cog.karajan.scheduler.TaskConstraints; +import org.globus.cog.karajan.scheduler.WeightedHost; import org.globus.cog.karajan.scheduler.WeightedHostScoreScheduler; import org.globus.cog.karajan.scheduler.WeightedHostSet; import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.util.Contact; import org.globus.cog.karajan.util.ContactSet; import org.globus.cog.karajan.util.TypeUtil; -import org.globus.swift.catalog.TCEntry; -import org.globus.swift.catalog.transformation.File; -import org.globus.swift.catalog.types.TCType; -import org.griphyn.vdl.util.FQN; +import org.globus.swift.catalog.site.SwiftContact; public class VDSAdaptiveScheduler extends WeightedHostScoreScheduler implements CoasterResourceTracker { @@ -61,7 +59,6 @@ private static Timer timer; - private TCCache tc; private LinkedList dq; private int clusteringQueueDelay = 1; private int minClusterTime = 60; @@ -98,9 +95,8 @@ public void setProperty(String name, Object value) { if (PROP_TC_FILE.equals(name)) { - tc = new TCCache(File.getNonSingletonInstance((String) value)); - this.setConstraintChecker(new TCChecker(tc)); - this.addTaskTransformer(new VDSTaskTransformer(tc)); + this.setConstraintChecker(new SwiftSiteChecker()); + this.addTaskTransformer(new VDSTaskTransformer()); } else if (PROP_CLUSTERING_QUEUE_DELAY.equals(name)) { clusteringQueueDelay = TypeUtil.toInt(value); @@ -115,7 +111,7 @@ super.setProperty(name, value); } } - + @Override public void enqueue(Task task, Object constraints, StatusListener l) { if (shouldBeClustered(task, constraints)) { @@ -465,23 +461,46 @@ } super.setResources(cs); for (BoundContact bc : cs.getContacts()) { - if ("passive".equals(bc.getProperty("globus:workerManager")) - && "true".equals(bc.getProperty("globus:throttleTracksWorkers"))) { - Service s = bc.getService(Service.EXECUTION, "coaster"); - if (s != null) { - s.setAttribute("resource-tracker", this); + Service es = bc.getService(Service.EXECUTION, "coaster"); + if (es != null && "passive".equals(es.getAttribute("workerManager")) + && "true".equals(bc.getProperty("throttleTracksWorkers"))) { + if (es != null) { + es.setAttribute("resource-tracker", this); WeightedHostSet whs = getWeightedHostSet(); // set throttle to one so that a task gets sent // to the provider and the connection/service is // initialized/started whs.changeThrottleOverride(whs.findHost(bc), 1); - serviceContactMapping.put(s, bc); + serviceContactMapping.put(es, bc); } } + + Object maxParallelJobs = bc.getProperty("maxParallelTasks"); + Object initialParallelJobs = bc.getProperty("initialParallelTasks"); + if (maxParallelJobs != null) { + double max = parseAndCheck(maxParallelJobs, "maxParallelTasks", bc); + bc.setProperty("jobThrottle", WeightedHost.jobThrottleFromMaxParallelism(max)); + if (initialParallelJobs != null) { + double initial = parseAndCheck(initialParallelJobs, "initialParallelTasks", bc); + bc.setProperty("initialScore", WeightedHost.initialScoreFromInitialParallelism(initial, max)); + } + } + else if (initialParallelJobs != null) { + throw new IllegalArgumentException("initialParallelJobs cannot be used without maxParallelTasks"); + } } } - public void resourceUpdated(Service service, String name, String value) { + private double parseAndCheck(Object value, String name, BoundContact bc) { + double d = TypeUtil.toDouble(value); + if (d < 1) { + throw new IllegalArgumentException("Invalid " + name + " value (" + d + ") for site '" + + bc.getName() + "'. Must be greater or equal to 1."); + } + return d; + } + + public void resourceUpdated(Service service, String name, String value) { if (logger.isInfoEnabled()) { logger.info(service + " resource updated: " + name + " -> " + value); } @@ -495,29 +514,13 @@ } } - public static class TCChecker implements ResourceConstraintChecker { - private TCCache tc; + public static class SwiftSiteChecker implements ResourceConstraintChecker { - public TCChecker(TCCache tc) { - this.tc = tc; - } - public boolean checkConstraints(BoundContact resource, TaskConstraints tc) { - if (isPresent("trfqn", tc)) { - FQN tr = (FQN) tc.getConstraint("trfqn"); - try { - List l = this.tc.getTCEntries(tr, resource.getHost(), TCType.INSTALLED); - if (l == null || l.isEmpty()) { - return false; - } - else { - return true; - } - } - catch (Exception e) { - logger.warn("Exception caught while querying TC", e); - return false; - } + if (isPresent("tr", tc)) { + SwiftContact sc = (SwiftContact) resource; + String tr = (String) tc.getConstraint("tr"); + return sc.findApplication(tr) != null; } else { return true; Modified: trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2014-07-04 06:33:48 UTC (rev 7951) +++ trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2014-07-04 06:34:38 UTC (rev 7952) @@ -41,8 +41,8 @@ private TaskTransformer impl; - public VDSTaskTransformer(TCCache tc) { - this.impl = new TCTransformer(tc); + public VDSTaskTransformer() { + this.impl = new SwiftTransformer(); } public void transformTask(Task task, Contact[] contacts, Service[] services) { @@ -54,7 +54,6 @@ public void transformTask(Task task, Contact[] contacts, Service[] services) { if (task.getType() == Task.JOB_SUBMISSION) { applyJobWorkDirectory(task, contacts); - applyTCEntry(task, contacts); } else if (task.getType() == Task.FILE_TRANSFER) { applyTransferWorkDirectory(task, contacts); @@ -158,21 +157,8 @@ } } - protected abstract void applyTCEntry(Task task, Contact[] contacts); } - public static class TCTransformer extends AbstractTransformer { - private TCCache tc; - - public TCTransformer(TCCache tc) { - this.tc = tc; - } - - protected void applyTCEntry(Task task, Contact[] contacts) { - // this method used to filter the task executable through - // tc.data, but the task executable at this point was - // always set to /bin/bash (or whatever the wrapper interpreter was). - // That was useless. - } + public static class SwiftTransformer extends AbstractTransformer { } } From hategan at ci.uchicago.edu Fri Jul 4 01:35:22 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:35:22 -0500 (CDT) Subject: [Swift-commit] r7953 - trunk/libexec Message-ID: <20140704063522.F00309DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:35:22 -0500 (Fri, 04 Jul 2014) New Revision: 7953 Modified: trunk/libexec/swift-int-staging.k Log: removed swift: prefix from properties Modified: trunk/libexec/swift-int-staging.k =================================================================== --- trunk/libexec/swift-int-staging.k 2014-07-04 06:34:38 UTC (rev 7952) +++ trunk/libexec/swift-int-staging.k 2014-07-04 06:35:22 UTC (rev 7953) @@ -82,9 +82,9 @@ swift:execute( progress, - siteProfile(rhost, "swift:wrapperInterpreter"), + siteProfile(rhost, "wrapperInterpreter"), list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), + siteProfile(rhost, "wrapperInterpreterOptions"), "_swiftwrap.staging", "-e", executable(tr, rhost), "-out", if(stdout == null, "stdout.txt", getFieldValue(stdout)), @@ -107,7 +107,7 @@ replicationChannel = replicationChannel jobid = jobid - stagingMethod := siteProfile(rhost, "swift:stagingMethod", default="proxy") + stagingMethod := siteProfile(rhost, "stagingMethod", default="proxy") stageIn("{PIN}{stagingMethod}://localhost/{SWIFT:HOME}/libexec/_swiftwrap.staging", "_swiftwrap.staging") From hategan at ci.uchicago.edu Fri Jul 4 01:37:54 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:37:54 -0500 (CDT) Subject: [Swift-commit] r7954 - trunk/libexec Message-ID: <20140704063754.63A3E9DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:37:53 -0500 (Fri, 04 Jul 2014) New Revision: 7954 Modified: trunk/libexec/ trunk/libexec/swift-int-wrapper-staging.k trunk/libexec/swift-int.k Log: removed swift: prefix from properties Property changes on: trunk/libexec ___________________________________________________________________ Modified: svn:mergeinfo - /branches/release-0.93/libexec:4761-5122 /branches/release-0.94/libexec:6283-6619,6998-7002,7019 /trunk/libexec:6172,6177,6182,6189-6190,6202-6203,6206-6208,6215-6223,6231-6241,6255-6258,6263,6272,6274-6618 + /branches/release-0.93/libexec:4761-5122 /branches/release-0.94/libexec:6283-6619,6998-7002,7019 /branches/release-0.95/libexec:7940 /trunk/libexec:6172,6177,6182,6189-6190,6202-6203,6206-6208,6215-6223,6231-6241,6255-6258,6263,6272,6274-6618 Modified: trunk/libexec/swift-int-wrapper-staging.k =================================================================== --- trunk/libexec/swift-int-wrapper-staging.k 2014-07-04 06:35:22 UTC (rev 7953) +++ trunk/libexec/swift-int-wrapper-staging.k 2014-07-04 06:37:53 UTC (rev 7954) @@ -68,9 +68,9 @@ log(LOG:INFO, "START dir={dir} host={host}") if(vdl:configProperty("sitedir.keep") == "false") { task:execute( - vdl:siteprofile(host, "swift:cleanupCommand"), + vdl:siteprofile(host, "cleanupCommand"), arguments = list( - siteProfile(host, "swift:cleanupCommandOptions"), + siteProfile(host, "cleanupCommandOptions"), dir ) host=host, batch=true, TCProfile(host) @@ -196,9 +196,9 @@ if (wrapperMode == "files") { swift:execute( progress, - siteProfile(rhost, "swift:wrapperInterpreter"), + siteProfile(rhost, "wrapperInterpreter"), list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), + siteProfile(rhost, "wrapperInterpreterOptions"), "_swiftwrap.wrapperstaging", jobid, "-urlprefix", URL_PREFIX, @@ -217,9 +217,9 @@ } if (wrapperMode == "args") { swift:execute( - siteProfile(rhost, "swift:wrapperInterpreter"), + siteProfile(rhost, "wrapperInterpreter"), list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), + siteProfile(rhost, "wrapperInterpreterOptions"), "_swiftwrap.wrapperstaging", jobid, "-urlprefix", URL_PREFIX, Modified: trunk/libexec/swift-int.k =================================================================== --- trunk/libexec/swift-int.k 2014-07-04 06:35:22 UTC (rev 7953) +++ trunk/libexec/swift-int.k 2014-07-04 06:37:53 UTC (rev 7954) @@ -83,7 +83,7 @@ log(LOG:INFO, "START host={rhost} - Initializing shared directory") dir:make(SHARED_DIR, host = rhost) - transfer(siteProfile(rhost, "swift:wrapperScript"), srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) + transfer(siteProfile(rhost, "wrapperScript"), srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) transfer("_swiftseq", srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) dir:make(dircat(RUN_DIR, "kickstart"), host=rhost) @@ -139,8 +139,8 @@ host=host, batch=true, TCProfile(host)) } if (swift:configProperty("sitedir.keep") == "false") { - task:execute(siteProfile(host, "swift:cleanupCommand"), - list(siteProfile(host, "swift:cleanupCommandOptions"), dir) + task:execute(siteProfile(host, "cleanupCommand"), + list(siteProfile(host, "cleanupCommandOptions"), dir) host=host, batch=true, TCProfile(host)) } log(LOG:INFO, "END dir={dir} host={host}") @@ -398,10 +398,10 @@ if (wrapperMode == "files") { swift:execute( progress, - siteProfile(rhost, "swift:wrapperInterpreter"), + siteProfile(rhost, "wrapperInterpreter"), list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), - dircat("shared", siteProfile(rhost, "swift:wrapperScript"), os=os), + siteProfile(rhost, "wrapperInterpreterOptions"), + dircat("shared", siteProfile(rhost, "wrapperScript"), os=os), jobid, "-p", jobdir ) directory = RUN_DIR @@ -416,10 +416,10 @@ else if (wrapperMode == "args") { swift:execute( progress, - siteProfile(rhost, "swift:wrapperInterpreter"), + siteProfile(rhost, "wrapperInterpreter"), list( - siteProfile(rhost, "swift:wrapperInterpreterOptions"), - dircat("shared", siteProfile(rhost, "swift:wrapperScript"), os=os), + siteProfile(rhost, "wrapperInterpreterOptions"), + dircat("shared", siteProfile(rhost, "wrapperScript"), os=os), jobid, "-jobdir", jobdir, "-scratch", try(siteProfile(rhost, "scratch"), "") From hategan at ci.uchicago.edu Fri Jul 4 01:38:29 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:38:29 -0500 (CDT) Subject: [Swift-commit] r7955 - trunk/src/org/griphyn/vdl/util Message-ID: <20140704063829.8B01B9DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:38:29 -0500 (Fri, 04 Jul 2014) New Revision: 7955 Removed: trunk/src/org/griphyn/vdl/util/FQN.java Log: removed FQN class Deleted: trunk/src/org/griphyn/vdl/util/FQN.java =================================================================== --- trunk/src/org/griphyn/vdl/util/FQN.java 2014-07-04 06:37:53 UTC (rev 7954) +++ trunk/src/org/griphyn/vdl/util/FQN.java 2014-07-04 06:38:29 UTC (rev 7955) @@ -1,158 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Nov 21, 2006 - */ -package org.griphyn.vdl.util; - -import java.util.StringTokenizer; - -/** - * - * Encapsulates a fully qualified name (namespace:name:version) - * - */ -public class FQN { - private final String namespace; - private final String name; - private final String version; - private final int hashCode; - - public FQN(String namespace, String name, String version) { - this.namespace = namespace; - this.name = name; - this.version = version; - this.hashCode = _hashCode(); - } - - public FQN(String namespace, String name) { - this(namespace, name, null); - } - - public FQN(String fqn) { - /* - * 1. name 2. namespace:name 3. :name:version 4. namespace:name:version - */ - if (fqn == null || fqn.length() == 0) { - throw new IllegalArgumentException("FQN is null/empty"); - } - String[] s = split(fqn); - if (fqn.charAt(0) == ':') { - // 3 - namespace = null; - name = s[0]; - version = s[1]; - } - else if (s.length == 1) { - // 1 - namespace = null; - name = s[0]; - version = null; - } - else if (s.length == 2) { - // 2 - namespace = s[0]; - name = s[1]; - version = null; - } - else if (s.length == 3) { - namespace = s[0]; - name = s[1]; - version = s[2]; - } - else { - throw new IllegalArgumentException("Invalid FQN: " + fqn); - } - this.hashCode = _hashCode(); - } - - private String[] split(String fqn) { - StringTokenizer st = new StringTokenizer(fqn, ":"); - String[] s = new String[st.countTokens()]; - for (int i = 0; i < s.length; i++) { - s[i] = st.nextToken(); - } - return s; - } - - public String getName() { - return name; - } - - public String getNamespace() { - return namespace; - } - - public String getVersion() { - return version; - } - - public String toString() { - if (namespace == null) { - if (version == null) { - return name; - } - else { - return ':' + name + ':' + version; - } - } - else { - if (version == null) { - return namespace + ':' + name; - } - else { - return namespace + ':' + name + ':' + version; - } - } - } - - public boolean equals(Object o) { - if (o instanceof FQN) { - FQN of = (FQN) o; - if (hashCode != of.hashCode) { - return false; - } - return cmpStr(namespace, of.namespace) && cmpStr(name, of.name) - && cmpStr(version, of.version); - } - else { - return false; - } - } - - private int _hashCode() { - int hc = 0; - hc += namespace == null ? 0 : namespace.hashCode(); - hc += name == null ? 0 : name.hashCode(); - hc += version == null ? 0 : version.hashCode(); - return hc; - } - - public int hashCode() { - return hashCode; - } - - private boolean cmpStr(String s1, String s2) { - if (s1 == null) { - return s2 == null; - } - else { - return s1.equals(s2); - } - } -} From hategan at ci.uchicago.edu Fri Jul 4 01:41:18 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:41:18 -0500 (CDT) Subject: [Swift-commit] r7956 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20140704064118.524029DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:41:17 -0500 (Fri, 04 Jul 2014) New Revision: 7956 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: added back ability to run .kml files with the swift command Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-04 06:38:29 UTC (rev 7955) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-04 06:41:17 UTC (rev 7956) @@ -154,6 +154,18 @@ } tree = load(project); } + else if (project.endsWith(".kml")) { + try { + tree = load(project); + } + catch (Exception pe) { + // the compiler should have already logged useful + // error messages, so this log line is just for + // debugging + logger.debug("Exception when compiling " + project, pe); + System.exit(3); + } + } else if (source != null) { try { String kml = compileString(source, provenanceEnabled); From hategan at ci.uchicago.edu Fri Jul 4 01:42:14 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:42:14 -0500 (CDT) Subject: [Swift-commit] r7957 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20140704064214.79BD79DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:42:14 -0500 (Fri, 04 Jul 2014) New Revision: 7957 Modified: trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java Log: changed collect list logging to debug Modified: trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java 2014-07-04 06:41:17 UTC (rev 7956) +++ trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java 2014-07-04 06:42:14 UTC (rev 7957) @@ -57,8 +57,8 @@ Collection stageouts = this.stageouts.getValue(stack); Collection collectList = this.collectList.getValue(stack); - if (logger.isInfoEnabled()) { - logger.info("Collect list: " + collectList); + if (logger.isDebugEnabled()) { + logger.debug("Collect list: " + collectList); } Channel log = cr_restartLog.get(stack); From hategan at ci.uchicago.edu Fri Jul 4 01:42:52 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:42:52 -0500 (CDT) Subject: [Swift-commit] r7958 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20140704064252.561D49DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:42:52 -0500 (Fri, 04 Jul 2014) New Revision: 7958 Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java Log: moved logging of heap usage and JVM threads to the runtime stats Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2014-07-04 06:42:14 UTC (rev 7957) +++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2014-07-04 06:42:52 UTC (rev 7958) @@ -340,6 +340,18 @@ Map summary = getSummary(); long now = System.currentTimeMillis(); + if (logger.isInfoEnabled()) { + Runtime r = Runtime.getRuntime(); + long maxHeap = r.maxMemory(); + long freeMemory = r.freeMemory(); + long totalMemory = r.totalMemory(); + long usedMemory = totalMemory - freeMemory; + int threadCount = Thread.activeCount(); + + logger.info("HeapMax: " + maxHeap + ", CrtHeap: " + totalMemory + ", UsedHeap: " + usedMemory + + ", JVMThreads: " + threadCount); + } + if (!forced && lastState != null && lastState.equals(summary)) { return false; } @@ -379,16 +391,6 @@ logger.info(msg); } - if (logger.isInfoEnabled()) { - Runtime r = Runtime.getRuntime(); - long maxHeap = r.maxMemory(); - long freeMemory = r.freeMemory(); - long totalMemory = r.totalMemory(); - long usedMemory = totalMemory - freeMemory; - - logger.info("HeapMax: " + maxHeap + ", CrtHeap: " + totalMemory + ", UsedHeap: " + usedMemory); - } - return true; } From hategan at ci.uchicago.edu Fri Jul 4 01:44:18 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:44:18 -0500 (CDT) Subject: [Swift-commit] r7959 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20140704064418.3354C9D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:44:17 -0500 (Fri, 04 Jul 2014) New Revision: 7959 Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetFieldConst.java trunk/src/org/griphyn/vdl/karajan/lib/New.java Log: some more compile-time optimizations Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetFieldConst.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/GetFieldConst.java 2014-07-04 06:42:52 UTC (rev 7958) +++ trunk/src/org/griphyn/vdl/karajan/lib/GetFieldConst.java 2014-07-04 06:44:17 UTC (rev 7959) @@ -24,23 +24,39 @@ import k.rt.Stack; import org.globus.cog.karajan.analyzer.ArgRef; -import org.globus.cog.karajan.analyzer.Param; -import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; +import org.globus.cog.karajan.analyzer.CompilationException; +import org.globus.cog.karajan.analyzer.Scope; +import org.globus.cog.karajan.analyzer.Signature; +import org.globus.cog.karajan.compiled.nodes.Node; +import org.globus.cog.karajan.compiled.nodes.functions.AbstractFunction; +import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.NoSuchTypeException; import org.griphyn.vdl.type.Types; -public class GetFieldConst extends AbstractSingleValuedFunction { +public class GetFieldConst extends AbstractFunction { private ArgRef name; private ArgRef type; + @Override + protected Signature getSignature() { + return new Signature(params("name", "type")); + } @Override - protected Param[] getParams() { - return params("name", "type"); + protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { + String name = this.name.getValue(); + String type = this.type.getValue(); + try { + staticReturn(scope, Field.Factory.getImmutableField(name, Types.getType(type))); + } + catch (NoSuchTypeException e) { + throw new ExecutionException("No such type: " + name, e); + } + + return null; } - @Override public Object function(Stack stack) { String name = this.name.getValue(stack); Modified: trunk/src/org/griphyn/vdl/karajan/lib/New.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/New.java 2014-07-04 06:42:52 UTC (rev 7958) +++ trunk/src/org/griphyn/vdl/karajan/lib/New.java 2014-07-04 06:44:17 UTC (rev 7959) @@ -22,7 +22,6 @@ import java.util.Collections; import java.util.List; -import java.util.Map; import k.rt.Context; import k.rt.ExecutionException; @@ -39,6 +38,7 @@ import org.globus.cog.karajan.parser.WrapperNode; import org.griphyn.vdl.mapping.DSHandle; import org.griphyn.vdl.mapping.DuplicateMappingChecker; +import org.griphyn.vdl.mapping.GenericMappingParamSet; import org.griphyn.vdl.mapping.InvalidMapperException; import org.griphyn.vdl.mapping.Mapper; import org.griphyn.vdl.mapping.MapperFactory; @@ -50,9 +50,6 @@ import org.griphyn.vdl.mapping.nodes.RootClosedArrayDataNode; import org.griphyn.vdl.mapping.nodes.RootClosedPrimitiveDataNode; import org.griphyn.vdl.mapping.nodes.RootFutureArrayDataNode; -import org.griphyn.vdl.mapping.nodes.RootFutureMappedSingleDataNode; -import org.griphyn.vdl.mapping.nodes.RootFuturePrimitiveDataNode; -import org.griphyn.vdl.mapping.nodes.RootFutureStructDataNode; import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.NoSuchTypeException; import org.griphyn.vdl.type.Type; @@ -63,15 +60,17 @@ private static final Mapper NULL_MAPPER = new NullMapper(); + private static DuplicateMappingChecker staticDMC = new DuplicateMappingChecker(null); + private ArgRef field; - private ArgRef> mapping; + private ArgRef mapping; private ArgRef value; private ArgRef waitCount; private ArgRef input; private ArgRef _defline; private VarRef context; - private VarRef cwd; + private VarRef cwd; @Override protected Signature getSignature() { @@ -103,7 +102,7 @@ public Object function(Stack stack) { Field field = this.field.getValue(stack); Object value = this.value.getValue(stack); - Map mapping = this.mapping.getValue(stack); + GenericMappingParamSet mapping = this.mapping.getValue(stack); Number waitCount = this.waitCount.getValue(stack); boolean input = this.input.getValue(stack); Integer line = this._defline.getValue(stack); @@ -117,14 +116,12 @@ String dbgname = (String) field.getId(); if (type.hasMappedComponents()) { - String desc = (String) mapping.remove("swift#descriptor"); try { - mapper = MapperFactory.getMapper(desc); + mapper = MapperFactory.getMapper(mapping.getDescriptor()); } catch (InvalidMapperException e) { - throw new ExecutionException(this, "Invalid mapper '" + desc + "'"); + throw new ExecutionException(this, "Invalid mapper '" + mapping.getDescriptor() + "'"); } - mapping.remove("descriptor"); mapper.setParameters(mapping); mapper.setBaseDir(cwd.getValue(stack)); } From hategan at ci.uchicago.edu Fri Jul 4 01:46:50 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:46:50 -0500 (CDT) Subject: [Swift-commit] r7960 - trunk/src/org/griphyn/vdl/mapping Message-ID: <20140704064650.2B5EC9D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:46:49 -0500 (Fri, 04 Jul 2014) New Revision: 7960 Modified: trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java Log: some previously uncommitted change Modified: trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2014-07-04 06:44:17 UTC (rev 7959) +++ trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2014-07-04 06:46:49 UTC (rev 7960) @@ -27,7 +27,12 @@ private final Map map; public DuplicateMappingChecker(VDL2Config conf) { - enabled = !"off".equals(conf.getProperty(VDL2ConfigProperties.DM_CHECKER)); + if (conf == null) { + enabled = true; + } + else { + enabled = !"off".equals(conf.getProperty(VDL2ConfigProperties.DM_CHECKER)); + } map = new HashMap(); } From hategan at ci.uchicago.edu Fri Jul 4 01:48:29 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:48:29 -0500 (CDT) Subject: [Swift-commit] r7961 - in trunk/src/org/griphyn/vdl/mapping: . file nodes Message-ID: <20140704064829.4586E9D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:48:28 -0500 (Fri, 04 Jul 2014) New Revision: 7961 Added: trunk/src/org/griphyn/vdl/mapping/GenericMappingParamSet.java Modified: trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java trunk/src/org/griphyn/vdl/mapping/DSHandle.java trunk/src/org/griphyn/vdl/mapping/Mapper.java trunk/src/org/griphyn/vdl/mapping/MappingParamSet.java trunk/src/org/griphyn/vdl/mapping/NullMapper.java trunk/src/org/griphyn/vdl/mapping/RootHandle.java trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedNonCompositeDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureNonCompositeDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureStructDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureArrayDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureMappedSingleDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureStructDataNode.java Log: some more mapper stuff cleaning Modified: trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -19,7 +19,6 @@ import java.util.Collection; import java.util.HashSet; -import java.util.Map; import java.util.Set; import org.apache.log4j.Logger; @@ -66,7 +65,7 @@ } @Override - public void setParameters(Map params) { + public void setParameters(GenericMappingParamSet params) { this.params = newParams(); this.params.setAll(params); } Modified: trunk/src/org/griphyn/vdl/mapping/DSHandle.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/DSHandle.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/DSHandle.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -84,6 +84,8 @@ public void closeShallow(); public void closeDeep(); + + public void closeArraySizes(); public Collection getFringePaths() throws HandleOpenException; Added: trunk/src/org/griphyn/vdl/mapping/GenericMappingParamSet.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/GenericMappingParamSet.java (rev 0) +++ trunk/src/org/griphyn/vdl/mapping/GenericMappingParamSet.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -0,0 +1,74 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 1, 2014 + */ +package org.griphyn.vdl.mapping; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import org.globus.cog.karajan.util.Pair; + +public class GenericMappingParamSet { + private final String descriptor; + private List> params; + + public GenericMappingParamSet(String descriptor) { + this.descriptor = descriptor; + } + + public void addParam(Pair param) { + if (params == null) { + params = new ArrayList>(4); + } + params.add(param); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append('<'); + sb.append(descriptor); + + if (params != null) { + sb.append("; "); + Iterator> it = params.iterator(); + while (it.hasNext()) { + Pair p = it.next(); + sb.append(p.s); + sb.append(" = "); + sb.append(p.t); + if (it.hasNext()) { + sb.append(", "); + } + } + } + sb.append(">"); + + return sb.toString(); + } + + public String getDescriptor() { + return descriptor; + } + + public Collection> getParams() { + if (params == null) { + return Collections.emptyList(); + } + else { + return params; + } + } + + public void put(String name, Object value) { + addParam(new Pair(name, value)); + } +} Modified: trunk/src/org/griphyn/vdl/mapping/Mapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/Mapper.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/Mapper.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -18,7 +18,6 @@ package org.griphyn.vdl.mapping; import java.util.Collection; -import java.util.Map; import java.util.Set; import org.griphyn.vdl.mapping.nodes.AbstractDataNode; @@ -83,7 +82,7 @@ Set getSupportedParamNames(); - void setParameters(Map params); + void setParameters(GenericMappingParamSet params); /** * Called after all parameters have been closed Modified: trunk/src/org/griphyn/vdl/mapping/MappingParamSet.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/MappingParamSet.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/MappingParamSet.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -4,8 +4,8 @@ package org.griphyn.vdl.mapping; import java.util.Collection; -import java.util.Map; +import org.globus.cog.karajan.util.Pair; import org.griphyn.vdl.karajan.lib.Tracer; import org.griphyn.vdl.mapping.nodes.AbstractDataNode; @@ -86,10 +86,10 @@ } } - public void setAll(Map m) { + public void setAll(GenericMappingParamSet m) { if (m != null) { - for (Map.Entry e : m.entrySet()) { - set0(e.getKey(), e.getValue()); + for (Pair e : m.getParams()) { + set0(e.s, e.t); } } } Modified: trunk/src/org/griphyn/vdl/mapping/NullMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/NullMapper.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/NullMapper.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -11,7 +11,6 @@ import java.util.Collection; import java.util.Collections; -import java.util.Map; import java.util.Set; import org.griphyn.vdl.mapping.nodes.AbstractDataNode; @@ -74,7 +73,7 @@ } @Override - public void setParameters(Map params) { + public void setParameters(GenericMappingParamSet params) { } @Override Modified: trunk/src/org/griphyn/vdl/mapping/RootHandle.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/RootHandle.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/RootHandle.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -25,7 +25,6 @@ Mapper getActualMapper(); boolean isArray(); - void closeArraySizes(); void mapperInitialized(Mapper mapper); } Modified: trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -20,8 +20,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -29,6 +27,7 @@ import org.griphyn.vdl.mapping.AbsFile; import org.griphyn.vdl.mapping.AbstractMapper; import org.griphyn.vdl.mapping.FileSystemLister; +import org.griphyn.vdl.mapping.GenericMappingParamSet; import org.griphyn.vdl.mapping.MappingParamSet; import org.griphyn.vdl.mapping.Path; import org.griphyn.vdl.mapping.PhysicalFormat; @@ -116,7 +115,7 @@ public static void main(String[] args) { RegularExpressionMapper reMapper = new RegularExpressionMapper(); - Map params = new HashMap(); + GenericMappingParamSet params = new GenericMappingParamSet("regex"); params.put("source", "2mass-j1223.fits"); params.put("match", "(.*)\\.(.*)"); params.put("transform", "\\1_area.\\2"); Modified: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedArrayDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedArrayDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -155,4 +155,11 @@ public int arraySize() { return values.length; } + + @Override + public void closeArraySizes() { + for (DSHandle h : values) { + h.closeArraySizes(); + } + } } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedNonCompositeDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedNonCompositeDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractClosedNonCompositeDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -39,4 +39,8 @@ public boolean isArray() { return false; } + + @Override + public void closeArraySizes() { + } } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureArrayDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -170,6 +170,19 @@ handles = Collections.emptyMap(); } } + + @Override + public void closeArraySizes() { + closeShallow(); + try { + for (DSHandle h : getAllFields()) { + h.closeArraySizes(); + } + } + catch (Exception e) { + throw new IllegalStateException(e); + } + } @Override public void closeDeep() { Modified: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureNonCompositeDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureNonCompositeDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureNonCompositeDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -91,4 +91,8 @@ value = null; super.clean0(); } + + @Override + public void closeArraySizes() { + } } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureStructDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureStructDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractFutureStructDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -139,4 +139,11 @@ throw exception; } } + + @Override + public void closeArraySizes() { + for (DSHandle h : fields) { + h.closeArraySizes(); + } + } } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/InitMapper.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -10,7 +10,6 @@ package org.griphyn.vdl.mapping.nodes; import java.util.Collection; -import java.util.Map; import java.util.Set; import k.rt.Future; @@ -24,6 +23,7 @@ import org.griphyn.vdl.mapping.DependentException; import org.griphyn.vdl.mapping.DuplicateMappingChecker; import org.griphyn.vdl.mapping.FileSystemLister; +import org.griphyn.vdl.mapping.GenericMappingParamSet; import org.griphyn.vdl.mapping.HandleOpenException; import org.griphyn.vdl.mapping.InvalidPathException; import org.griphyn.vdl.mapping.Mapper; @@ -98,7 +98,7 @@ } @Override - public void setParameters(Map params) { + public void setParameters(GenericMappingParamSet params) { } @Override Modified: trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureArrayDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureArrayDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureArrayDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -115,9 +115,4 @@ public Mapper getActualMapper() { return mapper; } - - @Override - public void closeArraySizes() { - closeShallow(); - } } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureMappedSingleDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureMappedSingleDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureMappedSingleDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -118,9 +118,4 @@ public Mapper getActualMapper() { return mapper; } - - @Override - public void closeArraySizes() { - // does not apply - } } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFuturePrimitiveDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -97,9 +97,4 @@ public Mapper getMapper() { throw new UnsupportedOperationException("Primitive types do not have a mapper"); } - - @Override - public void closeArraySizes() { - // does not apply - } } Modified: trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureStructDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureStructDataNode.java 2014-07-04 06:46:49 UTC (rev 7960) +++ trunk/src/org/griphyn/vdl/mapping/nodes/RootFutureStructDataNode.java 2014-07-04 06:48:28 UTC (rev 7961) @@ -129,9 +129,4 @@ public Mapper getActualMapper() { return mapper; } - - @Override - public void closeArraySizes() { - // does not apply - } } From hategan at ci.uchicago.edu Fri Jul 4 01:49:07 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:49:07 -0500 (CDT) Subject: [Swift-commit] r7962 - trunk/src/org/griphyn/vdl/type/impl Message-ID: <20140704064907.9EAA09D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:49:06 -0500 (Fri, 04 Jul 2014) New Revision: 7962 Modified: trunk/src/org/griphyn/vdl/type/impl/FieldImpl.java Log: better field.toString() Modified: trunk/src/org/griphyn/vdl/type/impl/FieldImpl.java =================================================================== --- trunk/src/org/griphyn/vdl/type/impl/FieldImpl.java 2014-07-04 06:48:28 UTC (rev 7961) +++ trunk/src/org/griphyn/vdl/type/impl/FieldImpl.java 2014-07-04 06:49:06 UTC (rev 7962) @@ -67,5 +67,9 @@ return o2 == null; } return o1.equals(o2); + } + + public String toString() { + return id + ": " + type; } } From hategan at ci.uchicago.edu Fri Jul 4 01:51:56 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:51:56 -0500 (CDT) Subject: [Swift-commit] r7963 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing Message-ID: <20140704065156.541FA9D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:51:55 -0500 (Fri, 04 Jul 2014) New Revision: 7963 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/GanttChart.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/GraphPanel.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SeriesWrapper.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SummaryPanel.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SwingMonitor.java Log: swing monitor updates Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/GanttChart.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/GanttChart.java 2014-07-04 06:49:06 UTC (rev 7962) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/GanttChart.java 2014-07-04 06:51:55 UTC (rev 7963) @@ -36,6 +36,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.TimerTask; import javax.swing.BoundedRangeModel; import javax.swing.JComponent; @@ -47,7 +48,6 @@ import javax.swing.JTable; import javax.swing.SpinnerNumberModel; import javax.swing.SwingConstants; -import javax.swing.Timer; import javax.swing.UIManager; import javax.swing.event.ChangeEvent; import javax.swing.event.ChangeListener; @@ -56,6 +56,7 @@ import org.globus.cog.abstraction.interfaces.Status; import org.globus.cog.abstraction.interfaces.Task; +import org.griphyn.vdl.karajan.monitor.SystemState; import org.griphyn.vdl.karajan.monitor.SystemStateListener; import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; import org.griphyn.vdl.karajan.monitor.items.StatefulItem; @@ -80,13 +81,14 @@ private JScrollBar hsb; private JSpinner scalesp; private long firstEvent; - private Timer timer; private double scale; private int offset, maxX; private JLabel ctime; private boolean scrollVerticallyOnNextUpdate; + private SystemState state; - public GanttChart() { + public GanttChart(SystemState state) { + this.state = state; scale = INITIAL_SCALE; jobs = new ArrayList(); jobmap = new HashMap(); @@ -127,8 +129,12 @@ add(createTools(), BorderLayout.NORTH); add(hsb, BorderLayout.SOUTH); - timer = new Timer(1000, this); - timer.start(); + state.schedule(new TimerTask() { + @Override + public void run() { + GanttChart.this.actionPerformed(null); + } + }, 1000, 1000); } private JComponent createTools() { @@ -166,7 +172,7 @@ public void itemUpdated(SystemStateListener.UpdateType updateType, StatefulItem item) { if (firstEvent == 0) { - firstEvent = System.currentTimeMillis(); + firstEvent = state.getCurrentTime(); } if (item.getItemClass().equals(StatefulItemClass.APPLICATION)) { ApplicationItem ai = (ApplicationItem) item; @@ -248,7 +254,7 @@ public void actionPerformed(ActionEvent e) { if (firstEvent != 0) { - ctime.setText("Current time: " + (System.currentTimeMillis() - firstEvent) / 1000 + "s"); + ctime.setText("Current time: " + (state.getCurrentTime() - firstEvent) / 1000 + "s"); } cmodel.fireTableDataChanged(); } @@ -428,7 +434,7 @@ public int time; public Event(int type) { - this.time = (int) (System.currentTimeMillis() - firstEvent); + this.time = (int) (state.getCurrentTime() - firstEvent); this.type = type; } } @@ -541,7 +547,7 @@ } if (!endcap) { - ex = (int) (System.currentTimeMillis() - firstEvent); + ex = (int) (state.getCurrentTime() - firstEvent); } g.setColor(LINE_COLOR); Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/GraphPanel.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/GraphPanel.java 2014-07-04 06:49:06 UTC (rev 7962) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/GraphPanel.java 2014-07-04 06:51:55 UTC (rev 7963) @@ -58,8 +58,6 @@ import org.jfree.chart.event.ChartChangeEvent; import org.jfree.chart.event.ChartChangeEventType; import org.jfree.chart.event.ChartChangeListener; -import org.jfree.chart.event.PlotChangeEvent; -import org.jfree.chart.event.PlotChangeListener; import org.jfree.chart.plot.XYPlot; import org.jfree.chart.renderer.xy.XYLineAndShapeRenderer; import org.jfree.data.Range; @@ -199,7 +197,12 @@ enableTooltip(x); } }; - GlobalTimer.getTimer().schedule(tooltipTimerTask, TOOLTIP_DISPLAY_DELAY); + try { + GlobalTimer.getTimer().schedule(tooltipTimerTask, TOOLTIP_DISPLAY_DELAY); + } + catch (IllegalStateException e) { + System.err.println(this + ": " + e.getMessage()); + } } protected synchronized void disableToolTip() { Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SeriesWrapper.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SeriesWrapper.java 2014-07-04 06:49:06 UTC (rev 7962) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SeriesWrapper.java 2014-07-04 06:51:55 UTC (rev 7963) @@ -221,7 +221,6 @@ return new Second(new Date(time)); } - @Override public Class getTimePeriodClass() { return Second.class; Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SummaryPanel.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SummaryPanel.java 2014-07-04 06:49:06 UTC (rev 7962) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SummaryPanel.java 2014-07-04 06:51:55 UTC (rev 7963) @@ -38,7 +38,7 @@ public SummaryPanel(SystemState state) { this.state = state; - this.start = System.currentTimeMillis(); + this.start = state.getCurrentTime(); SpringLayout l = new SpringLayout(); setLayout(l); @@ -161,7 +161,7 @@ } } long heapMax = state.getMaxHeap(); - long heapCrt = state.getCurrentHeap(); + long heapCrt = state.getUsedHeap(); memory.setMaximum((int) (heapMax / 1000000)); memory.setValue((int) (heapCrt / 1000000)); memory.setString(state.getCurrentHeapFormatted() + " / " + state.getMaxHeapFormatted()); Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SwingMonitor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SwingMonitor.java 2014-07-04 06:49:06 UTC (rev 7962) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/swing/SwingMonitor.java 2014-07-04 06:51:55 UTC (rev 7963) @@ -156,7 +156,7 @@ tablemap.put(StatefulItemClass.TASK, tasks); tabs.add("Tasks", (Component) tasks); - gantt = new GanttChart(); + gantt = new GanttChart(getState()); tabs.add("Gantt Chart", gantt); } From hategan at ci.uchicago.edu Fri Jul 4 01:52:33 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:52:33 -0500 (CDT) Subject: [Swift-commit] r7964 - trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi Message-ID: <20140704065233.47AFC9D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:52:32 -0500 (Fri, 04 Jul 2014) New Revision: 7964 Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/SummaryPane.java Log: ... Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/SummaryPane.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/SummaryPane.java 2014-07-04 06:51:55 UTC (rev 7963) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/ansi/SummaryPane.java 2014-07-04 06:52:32 UTC (rev 7964) @@ -97,7 +97,7 @@ // mem Runtime r = Runtime.getRuntime(); long heapMax = state.getMaxHeap(); - long heapCrt = state.getCurrentHeap(); + long heapCrt = state.getUsedHeap(); double fraction = (double) heapCrt / heapMax; memory.setValue((float) fraction); memory.setText(state.getCurrentHeapFormatted() + " / " + state.getMaxHeapFormatted()); From hategan at ci.uchicago.edu Fri Jul 4 01:54:52 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 01:54:52 -0500 (CDT) Subject: [Swift-commit] r7965 - in trunk/src/org/griphyn/vdl/karajan/monitor: . common items monitors/http processors processors/coasters processors/karajan processors/swift Message-ID: <20140704065452.039CC9D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 01:54:51 -0500 (Fri, 04 Jul 2014) New Revision: 7965 Added: trunk/src/org/griphyn/vdl/karajan/monitor/items/AbstractListenableStatefulItem.java trunk/src/org/griphyn/vdl/karajan/monitor/items/ChainedListener.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppDetailBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppInstanceBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppListBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppsSummaryBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/BrowserDataBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SiteInfoBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SwiftLogInfo.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/WorkerInfoBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/WorkerListBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerProbeItem.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerProbeProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppFailureProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/ConfigurationProcessor.java Modified: trunk/src/org/griphyn/vdl/karajan/monitor/MonitorAppender.java trunk/src/org/griphyn/vdl/karajan/monitor/StateUpdater.java trunk/src/org/griphyn/vdl/karajan/monitor/SystemState.java trunk/src/org/griphyn/vdl/karajan/monitor/common/DataSampler.java trunk/src/org/griphyn/vdl/karajan/monitor/items/AbstractStatefulItem.java trunk/src/org/griphyn/vdl/karajan/monitor/items/ApplicationItem.java trunk/src/org/griphyn/vdl/karajan/monitor/items/ApplicationState.java trunk/src/org/griphyn/vdl/karajan/monitor/items/StatefulItem.java trunk/src/org/griphyn/vdl/karajan/monitor/items/SummaryItem.java trunk/src/org/griphyn/vdl/karajan/monitor/items/TaskItem.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/HTTPServer.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/JSONEncoder.java trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SummaryDataBuilder.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/AbstractMessageProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/SimpleParser.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/BlockRequestedProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/CoasterStatusItem.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/RemoteLogProcessorDispatcher.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerActiveProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerLostProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerShutDownProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/karajan/TaskProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppEndProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppStartProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/JobProcessor.java trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java Log: monitor updates Modified: trunk/src/org/griphyn/vdl/karajan/monitor/MonitorAppender.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/MonitorAppender.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/MonitorAppender.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -40,15 +40,18 @@ import org.griphyn.vdl.karajan.monitor.processors.coasters.BlockFailedProcessor; import org.griphyn.vdl.karajan.monitor.processors.coasters.BlockRequestedProcessor; import org.griphyn.vdl.karajan.monitor.processors.coasters.RemoteLogProcessorDispatcher; +import org.griphyn.vdl.karajan.monitor.processors.coasters.WorkerActiveProcessor; import org.griphyn.vdl.karajan.monitor.processors.coasters.WorkerLostProcessor; +import org.griphyn.vdl.karajan.monitor.processors.coasters.WorkerProbeProcessor; import org.griphyn.vdl.karajan.monitor.processors.coasters.WorkerShutDownProcessor; -import org.griphyn.vdl.karajan.monitor.processors.coasters.WorkerActiveProcessor; import org.griphyn.vdl.karajan.monitor.processors.karajan.ExecutionContextProcessor; import org.griphyn.vdl.karajan.monitor.processors.karajan.SchedulerInfoProcessor; import org.griphyn.vdl.karajan.monitor.processors.karajan.TaskProcessor; import org.griphyn.vdl.karajan.monitor.processors.swift.AppEndProcessor; +import org.griphyn.vdl.karajan.monitor.processors.swift.AppFailureProcessor; import org.griphyn.vdl.karajan.monitor.processors.swift.AppStartProcessor; import org.griphyn.vdl.karajan.monitor.processors.swift.AppThreadProcessor; +import org.griphyn.vdl.karajan.monitor.processors.swift.ConfigurationProcessor; import org.griphyn.vdl.karajan.monitor.processors.swift.ForeachItEndProcessor; import org.griphyn.vdl.karajan.monitor.processors.swift.ForeachItStartProcessor; import org.griphyn.vdl.karajan.monitor.processors.swift.JobProcessor; @@ -92,11 +95,13 @@ updater.addProcessor(new JobProcessor()); updater.addProcessor(new SchedulerInfoProcessor()); updater.addProcessor(new ExecutionContextProcessor()); + updater.addProcessor(new ConfigurationProcessor()); addFilteredProcessors(updater, SwiftProcessorDispatcher.class, new AppStartProcessor(), new AppEndProcessor(), new AppThreadProcessor(), + new AppFailureProcessor(), new ProcedureStartProcessor(), new ProcedureEndProcessor(), new ForeachItStartProcessor(), @@ -110,7 +115,8 @@ new BlockFailedProcessor(), new WorkerActiveProcessor(), new WorkerLostProcessor(), - new WorkerShutDownProcessor()); + new WorkerShutDownProcessor(), + new WorkerProbeProcessor()); } private void addFilteredProcessors(StateUpdater updater, @@ -148,6 +154,7 @@ public void doAppend(LoggingEvent event) { try { + state.setCurrentTime(event.getTimeStamp()); updater.logEvent(event.getLevel(), event.getLogger().getName(), event.getMessage(), event.getThrowableInformation()); } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/StateUpdater.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/StateUpdater.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/StateUpdater.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -64,7 +64,6 @@ } public void logEvent(Object category, String source, Object message, Object details) { - state.setCurrentTime(System.currentTimeMillis()); Map> sources = levels.get(category); if (sources == null) { return; Modified: trunk/src/org/griphyn/vdl/karajan/monitor/SystemState.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/SystemState.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/SystemState.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -24,11 +24,12 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.SortedSet; import java.util.TimerTask; +import java.util.TreeSet; import k.rt.Stack; -import org.griphyn.vdl.karajan.monitor.common.GlobalTimer; import org.griphyn.vdl.karajan.monitor.items.StatefulItem; import org.griphyn.vdl.karajan.monitor.items.StatefulItemClass; import org.griphyn.vdl.karajan.monitor.items.SummaryItem; @@ -39,11 +40,13 @@ private Map> classes; private Set listeners; private Map stats; - private int total, completed, completedPreviously; - private long start, currentTime; + private int total, completed, completedPreviously, currentThreads, retries; + private long start, currentTime, usedHeap, maxHeap; private Stack stack; private String projectName; private final Runtime runtime; + private SortedSet tasks; + private boolean replicationEnabled, resumed; private static final Unit BYTES = new Unit.P2("B"); @@ -54,7 +57,8 @@ stats = new HashMap(); runtime = Runtime.getRuntime(); addItem(new SummaryItem()); - GlobalTimer.getTimer().schedule(new TimerTask() { + tasks = new TreeSet(); + schedule(new TimerTask() { public void run() { update(); } @@ -177,7 +181,7 @@ } public String getCurrentHeapFormatted() { - return BYTES.format(getCurrentHeap()); + return BYTES.format(getUsedHeap()); } public String getElapsedTimeFormatted() { @@ -248,18 +252,117 @@ public long getMaxHeap() { - return runtime.maxMemory(); + return maxHeap; } - public long getCurrentHeap() { - return runtime.totalMemory() - runtime.freeMemory(); + public long getUsedHeap() { + return usedHeap; } + public int getCurrentThreads() { + return currentThreads; + } + + public void setCurrentThreads(int currentThreads) { + this.currentThreads = currentThreads; + } + + public void setUsedHeap(long usedHeap) { + this.usedHeap = usedHeap; + } + + public void setMaxHeap(long maxHeap) { + this.maxHeap = maxHeap; + } + + public int getRetries() { + return retries; + } + + public void setRetries(int retries) { + this.retries = retries; + } + + public boolean getReplicationEnabled() { + return replicationEnabled; + } + + public void setReplicationEnabled(boolean replicationEnabled) { + this.replicationEnabled = replicationEnabled; + } + + public boolean getResumed() { + return resumed; + } + + public void setResumed(boolean resumed) { + this.resumed = resumed; + } + public long getCurrentTime() { return currentTime; } public void setCurrentTime(long currentTime) { this.currentTime = currentTime; + if (!tasks.isEmpty()) { + TimerTaskEntry e = tasks.first(); + while (e.nextTime < currentTime) { + tasks.remove(e); + + if (e.nextTime < MAX_INITIAL) { + e.nextTime = currentTime + e.nextTime; + } + else { + try { + e.task.run(); + } + catch (Exception ex) { + ex.printStackTrace(); + } + e.nextTime = e.nextTime + e.delay; + } + tasks.add(e); + e = tasks.first(); + } + } } + + private static class TimerTaskEntry implements Comparable { + public long nextTime; + public long delay; + public TimerTask task; + + public TimerTaskEntry(TimerTask task, long nextTime, long delay) { + this.task = task; + this.nextTime = nextTime; + this.delay = delay; + } + + @Override + public int compareTo(TimerTaskEntry o) { + long diff = nextTime - o.nextTime; + if (diff == 0) { + return System.identityHashCode(this) - System.identityHashCode(o); + } + else { + if (diff < 0) { + return -1; + } + else { + return 1; + } + } + } + } + + private static final long MAX_INITIAL = 1000000; + + public void schedule(TimerTask task, long initial, long repeat) { + if (initial > MAX_INITIAL) { + throw new IllegalArgumentException("Initial delay too large"); + } + TimerTaskEntry e = new TimerTaskEntry(task, currentTime + initial, repeat); + tasks.add(e); + } } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/common/DataSampler.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/common/DataSampler.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/common/DataSampler.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -55,7 +55,7 @@ this.listeners = new ArrayList(); initializeData(); - GlobalTimer.getTimer().schedule(new TimerTask() { + state.schedule(new TimerTask() { @Override public void run() { sample(); @@ -90,9 +90,9 @@ addSeries("Java Virtual Machine", new Series("jvm/heapUsed", "JVM Heap Used", BYTES, - new ReflectionSampler(state, "getCurrentHeap")), + new ReflectionSampler(state, "getUsedHeap")), new Series("jvm/activeThreads", "JVM Active Threads", COUNT, - new ReflectionSampler(Thread.class, "activeCount"))); + new ReflectionSampler(state, "getCurrentThreads"))); CoasterStatusItem coaster = (CoasterStatusItem) state.getItemByID(CoasterStatusItem.ID, StatefulItemClass.MISC); @@ -151,7 +151,7 @@ } protected void sample() { - long now = (System.currentTimeMillis() / 1000); + long now = state.getCurrentTime() / 1000; if (offset + count != now) { if (offset < 0) { offset = now; Added: trunk/src/org/griphyn/vdl/karajan/monitor/items/AbstractListenableStatefulItem.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/items/AbstractListenableStatefulItem.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/items/AbstractListenableStatefulItem.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,32 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 9, 2014 + */ +package org.griphyn.vdl.karajan.monitor.items; + +public abstract class AbstractListenableStatefulItem extends AbstractStatefulItem { + + private Listener listener; + + public AbstractListenableStatefulItem(String id) { + super(id); + } + + public void addListener(Listener listener) { + if (this.listener != null) { + listener = new ChainedListener(listener, this.listener); + } + this.listener = listener; + } + + protected void notifyListener() { + if (listener != null) { + listener.itemUpdated(this); + } + } +} Modified: trunk/src/org/griphyn/vdl/karajan/monitor/items/AbstractStatefulItem.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/items/AbstractStatefulItem.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/items/AbstractStatefulItem.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -76,4 +76,9 @@ public String getID() { return id; } + + @Override + public void addListener(Listener l) { + // not implemented by default + } } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/items/ApplicationItem.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/items/ApplicationItem.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/items/ApplicationItem.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -22,7 +22,7 @@ public class ApplicationItem extends AbstractStatefulItem { - private String name, arguments, host; + private String name, arguments, host, workerId; private long startTime, currentStateTime; /** * The state of the app. Currently swift does not log app state transitions @@ -97,6 +97,14 @@ return state; } + public String getWorkerId() { + return workerId; + } + + public void setWorkerId(String workerId) { + this.workerId = workerId; + } + public String toString() { return "APP[" + name + ", " + arguments + ", " + host + "]"; } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/items/ApplicationState.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/items/ApplicationState.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/items/ApplicationState.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -13,9 +13,14 @@ import java.util.List; public enum ApplicationState { + //0 INITIALIZING("Initializing"), SELECTING_SITE("Selecting site", "Sel. site"), STAGE_IN("Stage in"), + //3 SUBMITTING("Submitting"), SUBMITTED("Submitted"), ACTIVE("Active"), STAGE_OUT("Stage out"), - FAILED("Failed"), REPLICATING("Replicating"), FINISHED_IN_PREVIOUS_RUN("Finished in previous run", "Finished in prev. run", false), + //7 + FAILED("Failed"), REPLICATING("Replicating", "Replicating", false), + FINISHED_IN_PREVIOUS_RUN("Finished in previous run", "Finished in prev. run", false), + //10 FINISHED_SUCCESSFULLY("Finished successfully"); private String name, shortName; @@ -71,4 +76,8 @@ return enabledValues; } + + public boolean isTerminal() { + return this == FAILED || this == FINISHED_SUCCESSFULLY || this == FINISHED_IN_PREVIOUS_RUN; + } } \ No newline at end of file Added: trunk/src/org/griphyn/vdl/karajan/monitor/items/ChainedListener.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/items/ChainedListener.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/items/ChainedListener.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,25 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 9, 2014 + */ +package org.griphyn.vdl.karajan.monitor.items; + +public class ChainedListener implements StatefulItem.Listener { + private StatefulItem.Listener crt, old; + + public ChainedListener(StatefulItem.Listener crt, StatefulItem.Listener old) { + this.crt = crt; + this.old = old; + } + + @Override + public void itemUpdated(StatefulItem item) { + crt.itemUpdated(item); + old.itemUpdated(item); + } +} Modified: trunk/src/org/griphyn/vdl/karajan/monitor/items/StatefulItem.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/items/StatefulItem.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/items/StatefulItem.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -23,6 +23,10 @@ import java.util.Collection; public interface StatefulItem { + public interface Listener { + void itemUpdated(StatefulItem item); + } + StatefulItem getParent(); void setParent(StatefulItem parent); @@ -33,4 +37,6 @@ StatefulItemClass getItemClass(); String getID(); + + void addListener(Listener l); } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/items/SummaryItem.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/items/SummaryItem.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/items/SummaryItem.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -20,7 +20,6 @@ */ package org.griphyn.vdl.karajan.monitor.items; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -77,23 +76,11 @@ } public int getCount(String key, SystemState state) { - if (state.getStack() != null) { - // TODO Must get these from log - return -1; - } - else { - return getCount(key); - } + return getCount(key); } public synchronized Map getCounts(SystemState state) { - if (state.getStack() != null) { - // TODO Must get these from log - return Collections.emptyMap(); - } - else { - return new HashMap(counts); - } + return new HashMap(counts); } public synchronized void setCount(String key, int value) { Modified: trunk/src/org/griphyn/vdl/karajan/monitor/items/TaskItem.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/items/TaskItem.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/items/TaskItem.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -25,6 +25,7 @@ public class TaskItem extends AbstractStatefulItem { private Task task; private int status, type; + private String workerId; public TaskItem(String id, Task task) { super(id); @@ -76,4 +77,12 @@ public int getStatus() { return status; } + + public String getWorkerId() { + return workerId; + } + + public void setWorkerId(String workerdId) { + this.workerId = workerdId; + } } Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppDetailBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppDetailBuilder.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppDetailBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,243 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 29, 2014 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; +import org.griphyn.vdl.karajan.monitor.items.ApplicationState; +import org.griphyn.vdl.karajan.monitor.monitors.http.BrowserDataBuilder.TimedValue; + +public class AppDetailBuilder { + + private String name; + private SortedMap>> byName; + private BrowserDataBuilder db; + + public AppDetailBuilder(BrowserDataBuilder db, String name) { + this.db = db; + this.byName = db.getByName(); + this.name = name; + } + + private static class StateTimesAverage { + public static final int N = 7; + public int[] stateTimeSum = new int[N]; + public int[] stateTimeCount = new int[N]; + + public void add(ApplicationState state, long time) { + int index = INDEX_MAPPING[state.ordinal()]; + stateTimeCount[index]++; + stateTimeSum[index] += (int) time; + } + + public int[] getAverages() { + int[] avg = new int[N]; + for (int i = 0; i < N; i++) { + if (stateTimeCount[i] == 0) { + avg[i] = 0; + } + else { + avg[i] = stateTimeSum[i] / stateTimeCount[i]; + } + } + return avg; + } + } + + private static int[] INDEX_MAPPING = new int[ApplicationState.values().length]; + + static { + INDEX_MAPPING[ApplicationState.INITIALIZING.ordinal()] = 0; + INDEX_MAPPING[ApplicationState.SELECTING_SITE.ordinal()] = 1; + INDEX_MAPPING[ApplicationState.SUBMITTING.ordinal()] = 2; + INDEX_MAPPING[ApplicationState.SUBMITTED.ordinal()] = 3; + INDEX_MAPPING[ApplicationState.STAGE_IN.ordinal()] = 4; + INDEX_MAPPING[ApplicationState.ACTIVE.ordinal()] = 5; + INDEX_MAPPING[ApplicationState.STAGE_OUT.ordinal()] = 6; + } + + + public void getData(JSONEncoder e) { + // sites it ran on + Set sites = new TreeSet(); + // average times for each relevant state then for total + // relevant states: Initializing, Sel. site, Stage in, Submitting, Submitted (queued remotely). + // Active, Stage out, Total + StateTimesAverage st = new StateTimesAverage(); + // same for each site + Map sts = new HashMap(); + List totalTimesC = new ArrayList(); + List totalTimesF = new ArrayList(); + Map> totalTimesCompletedSite = new HashMap>(); + Map> totalTimesFailedSite = new HashMap>(); + + int count = 0; + List> l = byName.get(name); + for (int i = 0; i < l.size(); i++) { + SortedSet ss = l.get(i); + for (ApplicationItem item : ss) { + count++; + String host = item.getHost(); + if (!sites.contains(host)) { + sites.add(host); + sts.put(host, new StateTimesAverage()); + totalTimesCompletedSite.put(host, new ArrayList()); + totalTimesFailedSite.put(host, new ArrayList()); + } + + StateTimesAverage stss = sts.get(host); + List ttCs = totalTimesCompletedSite.get(host); + List ttFs = totalTimesFailedSite.get(host); + + List> tl = db.getTimeline(item); + long lastTime = -1; + long firstTime = -1; + ApplicationState lastState = null; + for (TimedValue p : tl) { + if (lastState != null) { + switch (lastState) { + case STAGE_IN: + firstTime = p.time; + case FINISHED_SUCCESSFULLY: + case FAILED: + case INITIALIZING: + case SELECTING_SITE: + case SUBMITTING: + case SUBMITTED: + case ACTIVE: + case STAGE_OUT: + st.add(lastState, p.time - lastTime); + stss.add(lastState, p.time - lastTime); + } + int time; + switch (lastState) { + case FINISHED_SUCCESSFULLY: + time = (int) (p.time - firstTime); + totalTimesC.add(time); + ttCs.add(time); + break; + case FAILED: + time = (int) (p.time - firstTime); + totalTimesF.add(time); + ttFs.add(time); + break; + } + } + lastTime = p.time; + lastState = p.value; + } + } + } + + // get range for total times + int minTime = Math.min(min(totalTimesC), min(totalTimesF)); + int maxTime = Math.max(max(totalTimesC), max(totalTimesF)); + + + int bins = (int) Math.max(Math.min(count / 5, 100.0), 1); + double binSize = ((double) (maxTime - minTime)) / bins; + + // now serialize this + e.beginMap(); + e.writeMapItem("name", name); + e.writeMapItem("count", count); + e.writeMapItem("completedCount", totalTimesC.size()); + e.writeMapItem("failedCount", totalTimesF.size()); + e.writeMapItem("avgStateTimes", st.getAverages()); + e.writeMapItem("distMinTime", minTime); + e.writeMapItem("distMaxTime", maxTime); + e.writeMapItem("bins", 100); + e.writeMapItem("completedTimeDist", bin(totalTimesC, binSize, minTime, maxTime, bins)); + e.writeMapItem("failedTimeDist", bin(totalTimesF, binSize, minTime, maxTime, bins)); + e.writeMapItem("completedTimeAvg", avg(totalTimesC)); + e.writeMapItem("failedTimeAvg", avg(totalTimesF)); + e.writeMapKey("sites"); + e.beginArray(); + for (String host : sites) { + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("name", host); + e.writeMapItem("count", totalTimesCompletedSite.get(host).size()); + e.writeMapItem("completedCount", totalTimesCompletedSite.get(host).size()); + e.writeMapItem("failedCount", totalTimesFailedSite.get(host).size()); + e.writeMapItem("avgStateTimes", sts.get(host).getAverages()); + e.writeMapItem("distMinTime", minTime); + e.writeMapItem("distMaxTime", maxTime); + e.writeMapItem("completedTimeDist", bin(totalTimesCompletedSite.get(host), binSize, minTime, maxTime, bins)); + e.writeMapItem("failedTimeDist", bin(totalTimesFailedSite.get(host), binSize, minTime, maxTime, bins)); + e.writeMapItem("completedTimeAvg", avg(totalTimesCompletedSite.get(host))); + e.writeMapItem("failedTimeAvg", avg(totalTimesFailedSite.get(host))); + e.endMap(); + e.endArrayItem(); + } + e.endArray(); + + e.endMap(); + } + + private int avg(List l) { + if (l.isEmpty()) { + return 0; + } + int sum = 0; + for (Integer i : l) { + sum += i; + } + return sum / l.size(); + } + + private List bin(List l, double binSize, int minTime, int maxTime, int binCount) { + List hist = new ArrayList(); + for (int i = 0; i < binCount; i++) { + hist.add(0); + } + for (Integer v : l) { + int bin = (int) Math.ceil((v - minTime) / binSize) - 1; + hist.set(bin, hist.get(bin) + 1); + } + return hist; + } + + private int min(List l) { + if (l.isEmpty()) { + return 0; + } + + int min = l.get(0); + for (Integer i : l) { + if (i < min) { + min = i; + } + } + return min; + } + + private int max(List l) { + if (l.isEmpty()) { + return 0; + } + + int max = l.get(0); + for (Integer i : l) { + if (i > max) { + max = i; + } + } + return max; + } +} Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppInstanceBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppInstanceBuilder.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppInstanceBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,148 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 29, 2014 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.globus.cog.abstraction.interfaces.JobSpecification; +import org.globus.cog.abstraction.interfaces.Task; +import org.griphyn.vdl.karajan.monitor.SystemState; +import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; +import org.griphyn.vdl.karajan.monitor.items.ApplicationState; +import org.griphyn.vdl.karajan.monitor.items.TaskItem; +import org.griphyn.vdl.karajan.monitor.monitors.http.BrowserDataBuilder.AppEntry; +import org.griphyn.vdl.karajan.monitor.monitors.http.BrowserDataBuilder.TimedValue; + +public class AppInstanceBuilder { + + private BrowserDataBuilder db; + private String id; + private Map entries; + private SystemState state; + + public AppInstanceBuilder(BrowserDataBuilder db, String id) { + this.db = db; + this.id = id; + this.entries = db.getEntries(); + this.state = db.getState(); + } + + public void getData(JSONEncoder e) { + AppEntry ae = entries.get(id); + if (ae == null) { + throw new IllegalArgumentException("Unknown application ID: " + id); + } + + ApplicationItem app = ae.item; + + e.beginMap(); + e.writeMapItem("id", id); + e.writeMapItem("name", app.getName()); + e.writeMapItem("args", app.getArguments()); + e.writeMapItem("host", app.getHost()); + e.writeMapItem("crtState", app.getState().ordinal()); + e.writeMapItem("totalTime", getTotalTime(ae.stateTimeline)); + e.writeMapItem("runTime", getRunTime(ae.stateTimeline)); + e.writeMapItem("timeline", db.getStateTimes(app)); + + if (app.getWorkerId() != null) { + e.writeMapItem("workerid", app.getWorkerId()); + } + + TaskItem et = null; + + if (ae.tasks != null) { + for (TaskItem it : ae.tasks) { + if (it.getType() == Task.JOB_SUBMISSION) { + et = it; + } + } + } + + if (et != null) { + Task t = et.getTask(); + JobSpecification spec = (JobSpecification) t.getSpecification(); + List args = spec.getArgumentsAsList(); + extractJobInfo(e, args); + e.writeMapItem("directory", spec.getDirectory()); + } + e.endMap(); + } + + + private int getTotalTime(List> tl) { + if (tl == null || tl.isEmpty()) { + return 0; + } + return (int) (state.getCurrentTime() - tl.get(0).time); + } + + private int getRunTime(List> tl) { + if (tl == null || tl.isEmpty()) { + return 0; + } + for (TimedValue p : tl) { + switch (p.value) { + case STAGE_IN: + case STAGE_OUT: + case ACTIVE: + return (int) (state.getCurrentTime() - p.time); + } + } + return 0; + } + + private void extractJobInfo(JSONEncoder e, List args) { + String key = null; + List l = new ArrayList(); + for (String arg : args) { + if (arg.startsWith("-")) { + if (key != null) { + writeJobInfoItem(e, key, l); + } + key = arg; + l.clear(); + } + else { + l.add(arg); + } + } + } + + private void writeJobInfoItem(JSONEncoder e, String key, List l) { + if (l.size() == 0) { + return; + } + if (key.equals("-e")) { + e.writeMapItem("executable", l.get(0)); + } + else if (key.equals("-out")) { + e.writeMapItem("stdout", l.get(0)); + } + else if (key.equals("-err")) { + e.writeMapItem("stderr", l.get(0)); + } + else if (key.equals("-i")) { + e.writeMapItem("stdin", l.get(0)); + } + else if (key.equals("-if")) { + e.writeMapItem("stagein", l); + } + else if (key.equals("-of")) { + e.writeMapItem("stageout", l); + } + else if (key.equals("-a")) { + e.writeMapItem("args", l); + } + } + +} Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppListBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppListBuilder.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppListBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,139 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 29, 2014 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.util.List; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; +import org.griphyn.vdl.karajan.monitor.items.ApplicationState; + +public class AppListBuilder { + + private String name; + private int page; + private int pageSize; + private int stateFilter; + private SortedMap>> byName; + private String hostFilter; + private BrowserDataBuilder db; + + public AppListBuilder(BrowserDataBuilder db, String name, int page, int pageSize, int state, String host) { + this.db = db; + this.byName = db.getByName(); + this.name = name; + if (name == null) { + this.name = ""; + } + this.page = page; + this.pageSize = pageSize; + this.stateFilter = state; + this.hostFilter = host; + } + + public void getData(JSONEncoder e) { + SortedSet sorted = getInstances(name, stateFilter, hostFilter); + + int start = (page - 1) * pageSize; + int index = 0; + e.beginMap(); + + String title; + if (stateFilter == -1) { + if (name.isEmpty()) { + title = "All application invocations"; + } + else { + title = "Invocations of application \"" + name + "\""; + } + } + else { + if (name.isEmpty()) { + title = ApplicationState.values()[stateFilter] + " application invocations"; + } + else { + title = ApplicationState.values()[stateFilter] + " invocations of application \"" + name + "\""; + } + } + if (hostFilter != null) { + title = title + " on site \"" + hostFilter + "\""; + } + e.writeMapItem("title", title); + db.writePagingData(e, sorted.size(), page, pageSize); + e.writeMapItem("name", name); + e.writeMapItem("state", stateFilter); + e.writeMapItem("host", hostFilter); + for (ApplicationItem item : sorted) { + if (index == start) { + e.writeMapKey("data"); + e.beginArray(); + } + if (index >= start) { + ApplicationState state = item.getState(); + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("id", item.getID()); + e.writeMapItem("state", state.ordinal()); + e.writeMapItem("startTime", item.getStartTime()); + e.writeMapItem("host", item.getHost()); + if (item.getWorkerId() != null) { + e.writeMapItem("worker", item.getWorkerId()); + } + e.writeMapItem("args", item.getArguments()); + if (state.isTerminal()) { + e.writeMapItem("runTime", item.getCurrentStateTime() - item.getStartTime()); + } + else { + e.writeMapItem("runTime", 0L); + } + e.endMap(); + e.endArrayItem(); + } + if (index > start + pageSize) { + e.endArray(); + e.endMap(); + return; + } + index++; + } + if (sorted.size() > 0) { + e.endArray(); + } + e.endMap(); + } + + private SortedSet getInstances(String name, int stateFilter, String hostFilter) { + SortedSet sorted = new TreeSet(BrowserDataBuilder.APP_TIME_COMPARATOR); + if (!name.isEmpty()) { + List> l = byName.get(name); + getInstances(sorted, l, stateFilter, hostFilter); + } + else { + for (List> l : byName.values()) { + getInstances(sorted, l, stateFilter, hostFilter); + } + } + return sorted; + } + + private void getInstances(SortedSet sorted, List> l, int stateFilter, String hostFilter) { + for (SortedSet ss : l) { + for (ApplicationItem app : ss) { + boolean stateMatch = stateFilter == -1 || app.getState().ordinal() == stateFilter; + boolean hostMatch = hostFilter == null || app.getHost().equals(hostFilter); + if (stateMatch && hostMatch) { + sorted.add(app); + } + } + } + } +} Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppsSummaryBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppsSummaryBuilder.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/AppsSummaryBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,55 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 29, 2014 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; + +import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; +import org.griphyn.vdl.karajan.monitor.items.ApplicationState; + +public class AppsSummaryBuilder { + + private final SortedMap>> byName; + private BrowserDataBuilder db; + + public AppsSummaryBuilder(BrowserDataBuilder db) { + this.byName = db.getByName(); + this.db = db; + } + + public void getData(JSONEncoder e) { + // counts of each state by name + e.beginMap(); + db.writeEnabledStates(e, "enabledStates"); + e.writeMapKey("apps"); + e.beginMap(); + for (Map.Entry>> en : byName.entrySet()) { + e.writeMapKey(en.getKey()); + e.beginArray(); + for (ApplicationState s : ApplicationState.values()) { + if (s.isEnabled()) { + e.beginArrayItem(); + e.beginArray(); + e.writeArrayItem(s.ordinal()); + e.writeArrayItem(en.getValue().get(s.ordinal()).size()); + e.endArray(); + e.endArrayItem(); + } + } + e.endArray(); + } + e.endMap(); + e.endMap(); + } + +} Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/BrowserDataBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/BrowserDataBuilder.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/BrowserDataBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,391 @@ +//---------------------------------------------------------------------- + //This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 21, 2013 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.griphyn.vdl.karajan.monitor.SystemState; +import org.griphyn.vdl.karajan.monitor.SystemStateListener; +import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; +import org.griphyn.vdl.karajan.monitor.items.ApplicationState; +import org.griphyn.vdl.karajan.monitor.items.StatefulItem; +import org.griphyn.vdl.karajan.monitor.items.StatefulItemClass; +import org.griphyn.vdl.karajan.monitor.items.TaskItem; +import org.griphyn.vdl.karajan.monitor.processors.coasters.CoasterStatusItem.Block; +import org.griphyn.vdl.karajan.monitor.processors.coasters.CoasterStatusItem; +import org.griphyn.vdl.karajan.monitor.processors.coasters.WorkerProbeItem; + +public class BrowserDataBuilder extends StateDataBuilder implements SystemStateListener { + + public static final Comparator APP_TIME_COMPARATOR = new Comparator() { + @Override + public int compare(ApplicationItem o1, ApplicationItem o2) { + return (int) (o1.getStartTime() - o2.getStartTime()); + } + }; + + public static class TimedValue { + public long time; + public T value; + + public TimedValue(long time, T value) { + this.time = time; + this.value = value; + } + } + + public static class AppEntry { + public ApplicationItem item; + public ApplicationState oldState; + public List tasks; + public List> stateTimeline; + } + + public static class WorkerData { + public String site; + public List cpuLoad; + public Map> diskUsage; + public Map> ioLoad; + public int activeApps; + public int failedApps; + public int completedApps; + + public WorkerData() { + cpuLoad = new ArrayList(); + diskUsage = new TreeMap>(); + ioLoad = new TreeMap>(); + } + } + + private final SystemState state; + private int maxCount; + private JSONEncoder e; + + private SortedMap>> byName; + private Map entries; + private List byTime; + + private SortedMap workerData; + private List workersByTime; + + public BrowserDataBuilder(SystemState state) { + this.state = state; + byName = new TreeMap>>(); + byTime = new ArrayList(); + entries = new HashMap(); + workerData = new TreeMap(); + workersByTime = new ArrayList(); + state.addListener(this); + } + + @Override + public void itemUpdated(UpdateType updateType, StatefulItem item) { + if (item.getItemClass() == StatefulItemClass.APPLICATION) { + appUpdated(updateType, (ApplicationItem) item); + } + else if (item.getItemClass() == StatefulItemClass.TASK) { + taskUpdated(updateType, (TaskItem) item); + } + else if (item instanceof WorkerProbeItem) { + addWorkerProbeData((WorkerProbeItem) item); + } + } + + private WorkerData getWorkerData(String id) { + WorkerData wd = workerData.get(id); + if (wd == null) { + wd = new WorkerData(); + workerData.put(id, wd); + workersByTime.add(id); + } + return wd; + } + + private void addWorkerProbeData(WorkerProbeItem item) { + WorkerData wd = getWorkerData(item.getID()); + WorkerProbeItem.Data data = item.getData(); + if (data instanceof WorkerProbeItem.Cpu) { + wd.cpuLoad.add((WorkerProbeItem.Cpu) data); + } + else if (data instanceof WorkerProbeItem.DiskUsage) { + WorkerProbeItem.DiskUsage du = (WorkerProbeItem.DiskUsage) data; + List l = wd.diskUsage.get(du.getMountPoint()); + if (l == null) { + l = new ArrayList(); + wd.diskUsage.put(du.getMountPoint(), l); + } + l.add(du); + } + else if (data instanceof WorkerProbeItem.IOLoad) { + WorkerProbeItem.IOLoad du = (WorkerProbeItem.IOLoad) data; + List l = wd.ioLoad.get(du.getDevice()); + if (l == null) { + l = new ArrayList(); + wd.ioLoad.put(du.getDevice(), l); + } + l.add(du); + } + } + + private void taskUpdated(UpdateType updateType, TaskItem task) { + ApplicationItem app = (ApplicationItem) task.getParent(); + if (app == null) { + return; + } + AppEntry e = entries.get(app.getID()); + if (e == null) { + return; + } + if (e.tasks == null) { + e.tasks = Collections.singletonList(task); + } + else if (e.tasks.size() == 1) { + if (!e.tasks.contains(task)) { + List l = new LinkedList(); + l.add(e.tasks.get(0)); + l.add(task); + e.tasks = l; + } + } + else { + if (!e.tasks.contains(task)) { + e.tasks.add(task); + } + } + } + + private void appUpdated(UpdateType updateType, ApplicationItem app) { + if (app.getName() == null) { + return; + } + if (entries.containsKey(app.getID())) { + updateApp(app); + } + else { + addApp(app); + } + } + + private void updateApp(ApplicationItem item) { + AppEntry e = entries.get(item.getID()); + ApplicationState old = e.oldState; + ApplicationState state = item.getState(); + e.oldState = state; + String name = item.getName(); + List> l = getNamed(name); + if (old != null) { + l.get(old.ordinal()).remove(item); + } + l.get(state.ordinal()).add(item); + List> timeline = getTimeline(e); + + timeline.add(new TimedValue(this.state.getCurrentTime(), state)); + + if (item.getWorkerId() != null) { + // initialize worker data + WorkerData wd = getWorkerData(item.getWorkerId()); + switch (state) { + case ACTIVE: + wd.activeApps++; + break; + case FAILED: + wd.failedApps++; + wd.activeApps--; + break; + case FINISHED_SUCCESSFULLY: + wd.activeApps--; + wd.completedApps++; + break; + } + wd.site = item.getHost(); + } + } + + private List> getTimeline(AppEntry e) { + List> tl = e.stateTimeline; + if (tl == null) { + tl = new ArrayList>(); + e.stateTimeline = tl; + } + return tl; + } + + List> getTimeline(ApplicationItem item) { + AppEntry e = entries.get(item.getID()); + if (e == null) { + throw new IllegalArgumentException("Unknown app id: " + item.getID()); + } + return getTimeline(e); + } + + private List> getNamed(String name) { + List> l = byName.get(name); + if (l == null) { + l = new ArrayList>(); + for (ApplicationState s : ApplicationState.values()) { + l.add(new TreeSet(APP_TIME_COMPARATOR)); + } + byName.put(name, l); + } + return l; + } + + private void addApp(ApplicationItem item) { + byTime.add(item); + String name = item.getName(); + if (name == null) { + return; + } + ApplicationState state = item.getState(); + AppEntry e = new AppEntry(); + entries.put(item.getID(), e); + e.item = item; + e.oldState = state; + List> l = getNamed(name); + l.get(state.ordinal()).add(item); + } + + @Override + public ByteBuffer getData(Map params) { + e = new JSONEncoder(); + String type = getParam(params, "type", "apps"); + + if (type.equals("apps")) { + new AppsSummaryBuilder(this).getData(e); + } + else if (type.equals("applist")) { + new AppListBuilder(this, getParam(params, "name", null), + Integer.parseInt(getParam(params, "page", "1")), + Integer.parseInt(getParam(params, "pageSize", "20")), + Integer.parseInt(getParam(params, "state", "-1")), + getParam(params, "host", null)).getData(e); + } + else if (type.equals("appdetail")) { + new AppDetailBuilder(this, getParam(params, "name")).getData(e); + } + else if (type.equals("appinstance")) { + new AppInstanceBuilder(this, getParam(params, "id")).getData(e); + } + else if (type.equals("sites")) { + new SiteInfoBuilder(this).getData(e); + } + else if (type.equals("workerlist")) { + new WorkerListBuilder(this, getParam(params, "site", null), + Integer.parseInt(getParam(params, "page", "1")), + Integer.parseInt(getParam(params, "pageSize", "20"))).getData(e); + } + else if (type.equals("worker")) { + new WorkerInfoBuilder(this, getParam(params, "id")).getData(e); + } + else { + throw new IllegalArgumentException("Unknown type: " + type); + } + return ByteBuffer.wrap(e.toString().getBytes()); + } + + public List> getStateTimes(ApplicationItem app) { + List> tl = getTimeline(app); + List> l = new ArrayList>(); + if (tl != null) { + long lastTime = -1; + long firstTime = -1; + ApplicationState lastState = null; + for (TimedValue p : tl) { + if (lastState != null) { + l.add(new org.griphyn.vdl.karajan.Pair(lastState.ordinal(), (int) (p.time - lastTime))); + } + lastTime = p.time; + lastState = p.value; + } + } + return l; + } + + + private String getParam(Map params, String name, String _default) { + String value = params.get(name); + if (value == null) { + return _default; + } + else { + return value; + } + } + + private String getParam(Map params, String name) { + String value = params.get(name); + if (value == null) { + throw new IllegalArgumentException("Missing parameter '" + name + "'"); + } + else { + return value; + } + } + + public SortedMap>> getByName() { + return byName; + } + + public Map getEntries() { + return entries; + } + + public SystemState getState() { + return state; + } + + public List getByTime() { + return byTime; + } + + public void writeEnabledStates(JSONEncoder e, String key) { + e.writeMapKey(key); + e.beginArray(); + for (ApplicationState s : ApplicationState.values()) { + if (s.isEnabled()) { + e.beginArrayItem(); + e.write(s.ordinal()); + } + } + e.endArray(); + } + + public Map getWorkerData() { + return workerData; + } + + public Block getBlock(String id) { + CoasterStatusItem item = (CoasterStatusItem) state.getItemByID(CoasterStatusItem.ID, StatefulItemClass.MISC); + return item.getBlocks().get(id); + } + + public List getWorkersByTime() { + return workersByTime; + } + + public void writePagingData(JSONEncoder e, int size, int page, int pageSize) { + int pages = (int) Math.ceil(((double) size) / pageSize); + e.writeMapItem("pages", pages); + e.writeMapItem("hasPrev", page > 1); + e.writeMapItem("hasNext", page < pages); + e.writeMapItem("crtPage", page); + } +} \ No newline at end of file Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/HTTPServer.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/HTTPServer.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/HTTPServer.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -69,6 +69,7 @@ stateKeys.put("/summary.state", new SummaryDataBuilder(state)); stateKeys.put("/plotSeriesInfo.state", new PlotInfoBuilder(state)); stateKeys.put("/plotData.state", new PlotDataBuilder(state)); + stateKeys.put("/browser.state", new BrowserDataBuilder(state)); } public void start() throws IOException { @@ -181,7 +182,19 @@ for (SelectionKey key : keys) { if (key.isValid()) { ConnectionState s = channels.get(key.channel()); - s.process(key); + boolean ok = false; + if (s != null) { + try { + s.process(key); + ok = true; + } + catch (Exception e) { + } + } + if (!ok) { + channels.remove(key.channel()); + key.cancel(); + } } } skeys.clear(); Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/JSONEncoder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/JSONEncoder.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/JSONEncoder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -9,6 +9,7 @@ */ package org.griphyn.vdl.karajan.monitor.monitors.http; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Stack; @@ -42,11 +43,28 @@ } public void write(String value) { - sb.append('"'); - sb.append(value); - sb.append('"'); + if (value == null) { + sb.append("null"); + } + else { + sb.append('"'); + escape(sb, value); + sb.append('"'); + } } + private void escape(StringBuilder sb, String value) { + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': + sb.append('\\'); + default: + sb.append(c); + } + } + } + public void write(boolean value) { sb.append(value); } @@ -73,14 +91,17 @@ else if (value instanceof Long) { write(((Long) value).longValue()); } - else if (value instanceof List) { - writeArray((List) value); + else if (value instanceof Collection) { + writeArray((Collection) value); } else if (value instanceof Map) { @SuppressWarnings("unchecked") Map m = (Map) value; writeMap(m); } + else if (value instanceof int[]) { + writeArray((int[]) value); + } else { write(value.toString()); } @@ -239,7 +260,7 @@ } } - public void writeArray(List a) { + public void writeArray(Collection a) { beginArray(); for (Object v : a) { writeArrayItem(v); Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SiteInfoBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SiteInfoBuilder.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SiteInfoBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,162 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 29, 2014 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; +import org.griphyn.vdl.karajan.monitor.items.ApplicationState; + +public class SiteInfoBuilder { + + private BrowserDataBuilder db; + private List byTime; + private List enabledStatesMapping; + + public SiteInfoBuilder(BrowserDataBuilder db) { + this.db = db; + this.byTime = db.getByTime(); + + enabledStatesMapping = new ArrayList(); + for (ApplicationState state : ApplicationState.values()) { + if (state.isEnabled()) { + enabledStatesMapping.add(state.ordinal()); + } + } + } + + private static class SiteInfo { + public String name; + public Map> appStates; + public Map> appCummulativeStateTimes; + public Map appCountByType; + public SortedSet workers; + public int appCount; + + public SiteInfo(String name) { + this.name = name; + this.appStates = new TreeMap>(); + this.appCountByType = new TreeMap(); + this.appCummulativeStateTimes = new TreeMap>(); + this.workers = new TreeSet(); + } + } + + + public void getData(JSONEncoder e) { + /* + * List of sites + * List of app names it ran with state counts and average times + * List of workers + */ + + Map si = new HashMap(); + + for (ApplicationItem app : byTime) { + addApp(getOrCreateSite(si, app.getHost()), app); + } + + e.beginMap(); + db.writeEnabledStates(e, "enabledStates"); + e.writeMapKey("sites"); + e.beginArray(); + for (SiteInfo i : si.values()) { + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("name", i.name); + e.writeMapItem("appCount", i.appCount); + e.writeMapItem("workers", i.workers); + e.writeMapKey("stateCounts"); + e.beginMap(); + for (Map.Entry> e1 : i.appStates.entrySet()) { + e.writeMapKey(e1.getKey()); + e.beginArray(); + int index = 0; + for (Integer i2 : e1.getValue()) { + if (ApplicationState.values()[index++].isEnabled()) { + e.writeArrayItem(i2.intValue()); + } + } + e.endArray(); + } + e.endMap(); + e.writeMapKey("avgStateTimes"); + e.beginMap(); + for (Map.Entry> e1 : i.appCummulativeStateTimes.entrySet()) { + e.writeMapKey(e1.getKey()); + int count = i.appCountByType.get(e1.getKey()); + e.beginArray(); + int index = 0; + for (Integer i2 : e1.getValue()) { + if (ApplicationState.values()[index++].isEnabled()) { + e.writeArrayItem(i2.intValue() / count); + } + } + e.endArray(); + } + e.endMap(); + e.endMap(); + } + e.endArray(); + e.endMap(); + } + + private void addApp(SiteInfo si, ApplicationItem app) { + si.appCount++; + if (app.getWorkerId() != null) { + si.workers.add(app.getWorkerId()); + } + String name = app.getName(); + + List states = si.appStates.get(name); + List stateTimes = si.appCummulativeStateTimes.get(name); + Integer count = si.appCountByType.get(name); + if (states == null) { + states = new ArrayList(); + states.addAll(Collections.nCopies(ApplicationState.values().length, 0)); + si.appStates.put(name, states); + + stateTimes = new ArrayList(); + stateTimes.addAll(Collections.nCopies(ApplicationState.values().length, 0)); + si.appCummulativeStateTimes.put(name, stateTimes); + + count = 0; + } + + add(states, app.getState().ordinal(), 1); + List> st = db.getStateTimes(app); + for (List sti : st) { + add(stateTimes, sti.get(0), sti.get(1)); + } + si.appCountByType.put(name, count + 1); + } + + private void add(List l, int index, int amount) { + int crt = l.get(index); + l.set(index, crt + amount); + } + + private SiteInfo getOrCreateSite(Map si, String host) { + SiteInfo s = si.get(host); + if (s == null) { + s = new SiteInfo(host); + si.put(host, s); + } + return s; + } + +} Modified: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SummaryDataBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SummaryDataBuilder.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SummaryDataBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -36,7 +36,7 @@ e.writeMapItem("completed", state.getCompleted()); e.writeMapItem("maxHeap", state.getMaxHeap()); e.writeMapItem("maxHeapFormatted", state.getMaxHeapFormatted()); - e.writeMapItem("crtHeap", state.getCurrentHeap()); + e.writeMapItem("crtHeap", state.getUsedHeap()); e.writeMapItem("crtHeapFormatted", state.getCurrentHeapFormatted()); e.writeMapItem("timeLeftFormatted", state.getEstimatedTimeLeftFormatted()); e.writeMapItem("elapsedTimeFormatetd", state.getElapsedTimeFormatted()); Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SwiftLogInfo.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SwiftLogInfo.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/SwiftLogInfo.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,139 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 1, 2014 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.text.ParseException; +import java.text.SimpleDateFormat; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.Priority; +import org.apache.log4j.spi.LoggingEvent; +import org.globus.cog.util.ArgumentParser; +import org.globus.cog.util.ArgumentParserException; +import org.griphyn.vdl.karajan.monitor.MonitorAppender; + +public class SwiftLogInfo { + private static SimpleDateFormat SDF = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSSZ"); + + public static final int FOLLOW_SLEEP_TIME = 50; + + private String logFileName; + private boolean follow; + private double rate; + + public SwiftLogInfo(String logFileName, boolean follow, double rate) { + this.logFileName = logFileName; + this.follow = follow; + this.rate = rate; + } + + public void run() throws Exception { + MonitorAppender ap = new MonitorAppender("bla", "http"); + BufferedReader br = new BufferedReader(new FileReader(logFileName)); + + long firstLogTime = -1; + long firstActualTime = System.currentTimeMillis(); + + if (follow) { + System.out.print("Following " + logFileName + ". Hit CTRL+C to end."); + } + else { + System.out.print("Parsing " + logFileName + "..."); + } + String line = null; + while (follow || (line = br.readLine()) != null) { + if (line == null) { + Thread.sleep(FOLLOW_SLEEP_TIME); + continue; + } + String[] els = line.split("\\s+", 5); + if (els.length < 5) { + continue; + } + + long time; + try { + time = SDF.parse(els[0] + " " + els[1]).getTime(); + } + catch (ParseException e) { + continue; + } + + if (rate != 0) { + if (firstLogTime == -1) { + firstLogTime = time; + } + long now = System.currentTimeMillis(); + // this event is supposed to happen at this relative time + long deadline = (long) ((time - firstLogTime) / rate); + long delay = deadline - (now - firstActualTime); + System.out.println("deadline: " + deadline + ", now: " + (now - firstActualTime)); + if (delay >= 0) { + Thread.sleep(delay); + } + } + + LoggingEvent le = new LoggingEvent(els[3], Logger.getLogger(els[3]), time, getLevel(els[2]), els[4], null); + ap.doAppend(le); + } + System.out.println("done"); + } + + public static void main(String[] args) { + ArgumentParser ap = new ArgumentParser(); + ap.addFlag("f", "Follow: keep parsing the log file as it grows."); + ap.addOption("rt", "integer", "Real time: if specified, " + + "generate log events progressively at a rate " + + "proportional to that at which they were generated.", + ArgumentParser.OPTIONAL); + ap.addOption(ArgumentParser.DEFAULT, "logFile", "The log file to parse", + ArgumentParser.NORMAL); + ap.addFlag("h", "Display usage information"); + + try { + ap.parse(args); + if (ap.isPresent("h")) { + ap.usage(); + System.exit(0); + } + SwiftLogInfo sli = new SwiftLogInfo(ap.getStringValue(ArgumentParser.DEFAULT), + ap.isPresent("f"), ap.getFloatValue("rt", 0)); + sli.run(); + } + catch (ArgumentParserException e) { + System.err.println(e.getMessage()); + ap.usage(); + } + catch (Exception e) { + e.printStackTrace(); + } + } + + private static Priority getLevel(String s) { + if ("WARN".equals(s)) { + return Level.WARN; + } + else if ("ERROR".equals(s)) { + return Level.ERROR; + } + else if ("INFO".equals(s)) { + return Level.INFO; + } + else if ("DEBUG".equals(s)) { + return Level.DEBUG; + } + else { + return Level.ALL; + } + } +} Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/WorkerInfoBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/WorkerInfoBuilder.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/WorkerInfoBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,116 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 30, 2014 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.util.List; +import java.util.Map; + +import org.griphyn.vdl.karajan.monitor.monitors.http.BrowserDataBuilder.WorkerData; +import org.griphyn.vdl.karajan.monitor.processors.coasters.CoasterStatusItem.Block; +import org.griphyn.vdl.karajan.monitor.processors.coasters.CoasterStatusItem.Worker; +import org.griphyn.vdl.karajan.monitor.processors.coasters.WorkerProbeItem; + +public class WorkerInfoBuilder { + + private BrowserDataBuilder db; + private String wid; + + public WorkerInfoBuilder(BrowserDataBuilder db, String wid) { + this.db = db; + this.wid = wid; + } + + public void getData(JSONEncoder e) { + /* + * worker, node running on, wall time, run time, apps running + * probes, all details + */ + e.beginMap(); + Map wd = db.getWorkerData(); + + WorkerData wdd = wd.get(wid); + + int index = wid.indexOf(':'); + String blkId = wid.substring(0, index); + String wId = wid.substring(index + 1); + + Block blk = db.getBlock(blkId); + Worker w = blk.getWorker(wId); + + e.writeMapItem("id", wid); + e.writeMapItem("node", w.node); + e.writeMapItem("cores", w.cores); + e.writeMapItem("startTime", blk.startTime); + e.writeMapItem("walltime", blk.walltime); + e.writeMapItem("activeApps", wdd.activeApps); + e.writeMapItem("failedApps", wdd.failedApps); + e.writeMapItem("completedApps", wdd.completedApps); + + e.writeMapKey("cpuLoad"); + e.beginArray(); + for (WorkerProbeItem.Cpu cpu : wdd.cpuLoad) { + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("t", cpu.getTime()); + e.writeMapItem("load", cpu.getLoad()); + e.endMap(); + } + e.endArray(); + + e.writeMapKey("diskUsage"); + e.beginArray(); + int ix = 0; + for (Map.Entry> e1 : wdd.diskUsage.entrySet()) { + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("index", ix++); + e.writeMapItem("mount", e1.getKey()); + e.writeMapKey("data"); + e.beginArray(); + for (WorkerProbeItem.DiskUsage du : e1.getValue()) { + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("t", du.getTime()); + e.writeMapItem("avail", du.getAvailable()); + e.writeMapItem("used", du.getUsed()); + e.endMap(); + } + e.endArray(); + e.endMap(); + } + e.endArray(); + + e.writeMapKey("ioLoad"); + e.beginArray(); + ix = 0; + for (Map.Entry> e1 : wdd.ioLoad.entrySet()) { + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("index", ix++); + e.writeMapItem("device", e1.getKey()); + e.writeMapKey("data"); + e.beginArray(); + for (WorkerProbeItem.IOLoad du : e1.getValue()) { + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("t", du.getTime()); + e.writeMapItem("rt", du.getReadThroughput()); + e.writeMapItem("wt", du.getWriteThroughput()); + e.writeMapItem("load", du.getLoad()); + e.endMap(); + } + e.endArray(); + e.endMap(); + } + e.endArray(); + + e.endMap(); + } +} Added: trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/WorkerListBuilder.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/WorkerListBuilder.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/monitors/http/WorkerListBuilder.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,106 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jun 30, 2014 + */ +package org.griphyn.vdl.karajan.monitor.monitors.http; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.griphyn.vdl.karajan.monitor.monitors.http.BrowserDataBuilder.WorkerData; +import org.griphyn.vdl.karajan.monitor.processors.coasters.CoasterStatusItem.Block; +import org.griphyn.vdl.karajan.monitor.processors.coasters.CoasterStatusItem.Worker; +import org.griphyn.vdl.karajan.monitor.processors.coasters.WorkerProbeItem; + +public class WorkerListBuilder { + + private BrowserDataBuilder db; + private int page; + private int pageSize; + private String site; + + public WorkerListBuilder(BrowserDataBuilder db, String site, int page, int pageSize) { + this.db = db; + this.site = site; + this.page = page; + this.pageSize = pageSize; + } + + public void getData(JSONEncoder e) { + /* + * worker, node running on, wall time, run time, #apps running + * probes + */ + Map wd = db.getWorkerData(); + List filtered = new ArrayList(); + for (String wid : db.getWorkersByTime()) { + WorkerData wdd = wd.get(wid); + if (site == null || site.equals(wdd.site)) { + filtered.add(wid); + } + } + int start = (page - 1) * pageSize; + int i = -1; + e.beginMap(); + db.writePagingData(e, filtered.size(), page, pageSize); + e.writeMapKey("data"); + e.beginArray(); + for (String wid : filtered) { + i++; + if (i < start) { + continue; + } + if (i > start + pageSize) { + break; + } + WorkerData wdd = wd.get(wid); + + e.beginArrayItem(); + e.beginMap(); + + e.writeMapItem("id", wid); + int index = wid.indexOf(':'); + String blkId = wid.substring(0, index); + String wId = wid.substring(index + 1); + + Block blk = db.getBlock(blkId); + Worker w = blk.getWorker(wId); + + e.writeMapItem("node", w.node); + e.writeMapItem("cores", w.cores); + e.writeMapItem("startTime", blk.startTime); + e.writeMapItem("walltime", blk.walltime); + e.writeMapItem("activeApps", wdd.activeApps); + e.writeMapItem("failedApps", wdd.failedApps); + e.writeMapItem("completedApps", wdd.completedApps); + + e.writeMapKey("cpuLoad"); + e.beginArray(); + for (WorkerProbeItem.Cpu cpu : wdd.cpuLoad) { + e.beginArrayItem(); + e.beginMap(); + e.writeMapItem("t", cpu.getTime()); + e.writeMapItem("l", cpu.getLoad()); + e.endMap(); + } + e.endArray(); + e.endMap(); + } + e.endArray(); + e.endMap(); + } + + private T getLast(List l) { + if (l == null || l.isEmpty()) { + return null; + } + return l.get(l.size() - 1); + } + +} Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/AbstractMessageProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/AbstractMessageProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/AbstractMessageProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -17,7 +17,8 @@ @Override public String getSupportedSourceName() { - return getSupportedSource().getName(); + String name = getSupportedSource().getName(); + return name.substring(name.lastIndexOf('.') + 1); } @Override Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/SimpleParser.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/SimpleParser.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/SimpleParser.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -49,6 +49,25 @@ crt = index + tok.length(); } + public void markMatchedTo(char m, char pair) throws ParsingException { + int level = 1; + for (int i = crt; i < str.length(); i++) { + char c = str.charAt(i); + if (c == m) { + level--; + if (level == 0) { + tokEnd = i; + return; + } + } + if (c == pair) { + level++; + } + } + throw new ParsingException("Could not find \"" + m + "\" in \"" + remaining() + + "\". String is \"" + str + "\"."); + } + public void skipTo(String tok) throws ParsingException { int index = str.indexOf(tok, crt); if (index == -1) { Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/BlockRequestedProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/BlockRequestedProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/BlockRequestedProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -41,9 +41,11 @@ p.beginToken(); p.markTo(","); int coresPerWorker = Integer.parseInt(p.getToken()); + p.skip("walltime="); + int walltime = Integer.parseInt(p.remaining()); CoasterStatusItem item = (CoasterStatusItem) state.getItemByID(CoasterStatusItem.ID, StatefulItemClass.MISC); - item.newBlock(blockId, cores, coresPerWorker); + item.newBlock(blockId, cores, coresPerWorker, walltime); } catch (Exception e) { e.printStackTrace(); Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/CoasterStatusItem.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/CoasterStatusItem.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/CoasterStatusItem.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -33,8 +33,8 @@ return StatefulItemClass.MISC; } - public synchronized void newBlock(String id, int cores, int coresPerWorker) { - Block b = new Block(id, cores, coresPerWorker); + public synchronized void newBlock(String id, int cores, int coresPerWorker, int walltime) { + Block b = new Block(id, cores, coresPerWorker, walltime); blocks.put(id, b); queuedBlocks++; requestedCores += cores; @@ -69,25 +69,31 @@ doneBlocks++; } - public synchronized void workerActive(String blockId) { + public synchronized void workerActive(String blockId, String workerId, String node, int cores, long now) { Block b = getBlock(blockId); - b.activeCores += b.coresPerWorker; - activeCores += b.coresPerWorker; - requestedCores -= b.coresPerWorker; + activeCores += cores; + requestedCores -= cores; + b.addWorker(workerId, node, cores, now); } - public synchronized void workerLost(String blockId) { + public synchronized void workerLost(String blockId, String workerId) { Block b = getBlock(blockId); - b.activeCores -= b.coresPerWorker; - activeCores -= b.coresPerWorker; - failedCores += b.coresPerWorker; + Worker w = b.getWorker(workerId); + int cores = w.cores; + activeCores -= cores; + failedCores += cores; + b.activeCores -= cores; + w.state = WorkerState.FAILED; } - public synchronized void workerShutDown(String blockId) { + public synchronized void workerShutDown(String blockId, String workerId) { Block b = getBlock(blockId); - b.activeCores -= b.coresPerWorker; - activeCores -= b.coresPerWorker; - doneCores += b.coresPerWorker; + Worker w = b.getWorker(workerId); + int cores = w.cores; + activeCores -= cores; + failedCores += cores; + b.activeCores -= cores; + w.state = WorkerState.FAILED; } private Block getBlock(String blockId) { @@ -148,17 +154,53 @@ QUEUED, ACTIVE, FAILED, DONE } + public enum WorkerState { + ACTIVE, FAILED, DONE + } + + public static class Worker { + public String node; + public int cores; + public WorkerState state; + + public Worker(String node, int cores) { + this.node = node; + this.cores = cores; + } + } + public static class Block { public BlockState state; public final String id; public final int cores, coresPerWorker; public int activeCores; + public int walltime; + public long startTime; + public Map workers; - public Block(String id, int cores, int coresPerWorker) { + public Block(String id, int cores, int coresPerWorker, int walltime) { this.id = id; this.cores = cores; this.coresPerWorker = coresPerWorker; this.state = BlockState.QUEUED; + this.walltime = walltime; } + + public Worker getWorker(String workerId) { + return workers.get(workerId); + } + + public void addWorker(String workerId, String node, int cores, long now) { + if (workers == null) { + workers = new HashMap(); + } + Worker w = new Worker(node, cores); + w.state = WorkerState.ACTIVE; + workers.put(workerId, w); + activeCores += cores; + if (startTime == 0) { + startTime = now; + } + } } } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/RemoteLogProcessorDispatcher.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/RemoteLogProcessorDispatcher.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/RemoteLogProcessorDispatcher.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -20,7 +20,7 @@ @Override public String getSupportedSourceName() { - return RemoteLogHandler.class.getName(); + return RemoteLogHandler.class.getSimpleName(); } @Override Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerActiveProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerActiveProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerActiveProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -31,9 +31,15 @@ try { p.skip("blockid="); String blockId = p.word(); + p.skip("id="); + String workerId = p.word(); + p.skip("node="); + String node = p.word(); + p.skip("cores="); + int cores = Integer.parseInt(p.word()); CoasterStatusItem item = (CoasterStatusItem) state.getItemByID(CoasterStatusItem.ID, StatefulItemClass.MISC); - item.workerActive(blockId); + item.workerActive(blockId, workerId, node, cores, state.getCurrentTime()); } catch (Exception e) { e.printStackTrace(); Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerLostProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerLostProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerLostProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -31,9 +31,11 @@ try { p.skip("blockid="); String blockId = p.word(); + p.skip("id="); + String workerId = p.word(); CoasterStatusItem item = (CoasterStatusItem) state.getItemByID(CoasterStatusItem.ID, StatefulItemClass.MISC); - item.workerLost(blockId); + item.workerLost(blockId, workerId); } catch (Exception e) { e.printStackTrace(); Added: trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerProbeItem.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerProbeItem.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerProbeItem.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,151 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Aug 7, 2013 + */ +package org.griphyn.vdl.karajan.monitor.processors.coasters; + +import java.util.Collection; + +import org.griphyn.vdl.karajan.monitor.items.StatefulItem; +import org.griphyn.vdl.karajan.monitor.items.StatefulItemClass; + +public class WorkerProbeItem implements StatefulItem { + private String workerid; + private final Data data; + + public WorkerProbeItem(String workerid, Data data) { + this.workerid = workerid; + this.data = data; + } + + public Data getData() { + return data; + } + + + + public static class Data { + private final long time; + + public Data(long time) { + this.time = time; + } + + public long getTime() { + return time; + } + } + + public static class Cpu extends Data { + private final double load; + + public Cpu(long time, double load) { + super(time); + this.load = load; + } + + public double getLoad() { + return load; + } + } + + public static class DiskUsage extends Data { + private final String mountPoint, fs; + private final long used, available; + + public DiskUsage(long time, String mountPoint, String fs, long used, long available) { + super(time); + this.mountPoint = mountPoint; + this.fs = fs; + this.used = used; + this.available = available; + } + + public String getMountPoint() { + return mountPoint; + } + + public String getFs() { + return fs; + } + + public long getUsed() { + return used; + } + + public long getAvailable() { + return available; + } + } + + public static class IOLoad extends Data { + private final String device; + private final int writeThroughput, readThroughput; + private final double load; + + public IOLoad(long time, String device, int writeThroughput, int readThroughput, double load) { + super(time); + this.device = device; + this.writeThroughput = writeThroughput; + this.readThroughput = readThroughput; + this.load = load; + } + + public String getDevice() { + return device; + } + + public long getWriteThroughput() { + return writeThroughput; + } + + public long getReadThroughput() { + return readThroughput; + } + + public double getLoad() { + return load; + } + } + + @Override + public StatefulItem getParent() { + return null; + } + + @Override + public void setParent(StatefulItem parent) { + } + + @Override + public void addChild(StatefulItem child) { + } + + @Override + public void removeChild(StatefulItem child) { + } + + @Override + public Collection getChildren() { + return null; + } + + @Override + public StatefulItemClass getItemClass() { + return StatefulItemClass.MISC; + } + + @Override + public String getID() { + return workerid; + } + + @Override + public void addListener(Listener l) { + } +} Added: trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerProbeProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerProbeProcessor.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerProbeProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,76 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Aug 7, 2013 + */ +package org.griphyn.vdl.karajan.monitor.processors.coasters; + +import org.griphyn.vdl.karajan.monitor.SystemState; +import org.griphyn.vdl.karajan.monitor.items.StatefulItem; +import org.griphyn.vdl.karajan.monitor.processors.SimpleParser; + +public class WorkerProbeProcessor extends AbstractRemoteLogProcessor { + private CoasterStatusItem item; + + @Override + public void initialize(SystemState state) { + super.initialize(state); + } + + @Override + public String getMessageHeader() { + return "PROBE"; + } + + @Override + public void processMessage(SystemState state, SimpleParser p, Object details) { + try { + p.skip("type="); + String type = p.word(); + p.skip("workerid="); + String workerid = p.word(); + p.skip("time="); + long time = (long) (1000 * Double.parseDouble(p.word())); + + StatefulItem item = null; + + if (type.equals("CPU")) { + p.skip("load="); + double load = Double.parseDouble(p.word()); + item = new WorkerProbeItem(workerid, new WorkerProbeItem.Cpu(time, load)); + } + else if (type.equals("DF")) { + p.skip("mount="); + String mountPoint = p.word(); + p.skip("fs="); + String fs = p.word(); + p.skip("used="); + long usedBytes = 1024 * Long.parseLong(p.word()); + p.skip("avail="); + long availBytes = 1024 * Long.parseLong(p.word()); + item = new WorkerProbeItem(workerid, new WorkerProbeItem.DiskUsage(time, mountPoint, fs, usedBytes, availBytes)); + } + else if (type.equals("DL")) { + p.skip("dev="); + String device = p.word(); + p.skip("wtr="); + int wtr = (int) Double.parseDouble(p.word()); + p.skip("rtr="); + int rtr = (int) Double.parseDouble(p.word()); + p.skip("load="); + double load = Double.parseDouble(p.word()); + item = new WorkerProbeItem(workerid, new WorkerProbeItem.IOLoad(time, device, wtr, rtr, load)); + } + if (item != null) { + state.itemUpdated(item); + } + } + catch (Exception e) { + e.printStackTrace(); + } + } +} Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerShutDownProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerShutDownProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/coasters/WorkerShutDownProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -31,9 +31,11 @@ try { p.skip("blockid="); String blockId = p.word(); + p.skip("id="); + String workerId = p.word(); CoasterStatusItem item = (CoasterStatusItem) state.getItemByID(CoasterStatusItem.ID, StatefulItemClass.MISC); - item.workerShutDown(blockId); + item.workerShutDown(blockId, workerId); } catch (Exception e) { e.printStackTrace(); Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/karajan/TaskProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/karajan/TaskProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/karajan/TaskProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -21,6 +21,9 @@ package org.griphyn.vdl.karajan.monitor.processors.karajan; import org.apache.log4j.Level; +import org.globus.cog.abstraction.impl.common.task.JobSpecificationImpl; +import org.globus.cog.abstraction.impl.common.task.TaskImpl; +import org.globus.cog.abstraction.interfaces.JobSpecification; import org.globus.cog.abstraction.interfaces.Status; import org.globus.cog.abstraction.interfaces.Task; import org.griphyn.vdl.karajan.monitor.SystemState; @@ -100,6 +103,42 @@ } } } + else if (p.matchAndSkip("JOB_TASK ")) { + p.skip("jobid="); + String jobid = p.word(); + p.skip("taskid="); + id = p.word(); + p.skip("exec="); + String exec = p.word(); + p.skip("dir="); + String dir = p.word(); + p.skip("args="); + String args = p.remaining(); + + ti = new TaskItem(id, Task.JOB_SUBMISSION); + Task t = new TaskImpl(); + t.setType(Task.JOB_SUBMISSION); + JobSpecification spec = new JobSpecificationImpl(); + spec.setExecutable(exec); + spec.setArguments(args); + spec.setDirectory(dir); + t.setSpecification(spec); + ti.setTask(t); + updateParent(state, id, ti); + state.addItem(ti); + } + else if (p.matchAndSkip("TASK_STATUS_CHANGE ")) { + p.skip("taskid="); + id = p.word(); + p.skip("status="); + taskState = Integer.parseInt(p.word()); + ti = (TaskItem) state.getItemByID(id, StatefulItemClass.TASK); + if (p.matchAndSkip("workerid=")) { + ti.setWorkerId(p.word()); + } + ti.setStatus(taskState); + state.itemUpdated(ti); + } else if (p.matchAndSkip("Task(")) { p.skip("type="); p.beginToken(); @@ -117,6 +156,7 @@ } else { ti = new TaskItem(id, taskType); + updateParent(state, id, ti); state.addItem(ti); } } @@ -125,6 +165,23 @@ e.printStackTrace(); } } + if (ti != null && ti.getParent() != null && taskState != 0) { + ApplicationItem app = (ApplicationItem) ti.getParent(); + switch (taskState) { + case Status.SUBMITTING: + case Status.SUBMITTED: + case Status.ACTIVE: + case Status.STAGE_IN: + case Status.STAGE_OUT: + app.setState(getAppStateFromTaskState(taskState), state.getCurrentTime()); + app.setWorkerId(ti.getWorkerId()); + state.itemUpdated(app); + break; + } + } + } + + private void updateParent(SystemState state, String id, TaskItem ti) { if (ti != null && id != null && ti.getParent() == null) { int bi = id.indexOf(':'); int li = id.lastIndexOf('-'); @@ -138,20 +195,22 @@ bridge.addChild(ti); } } - if (ti != null && ti.getParent() != null && taskState != 0) { - ApplicationItem app = (ApplicationItem) ti.getParent(); - if (taskState == Status.SUBMITTING) { - app.setState(ApplicationState.SUBMITTING, state.getCurrentTime()); - state.itemUpdated(app); - } - else if (taskState == Status.SUBMITTED) { - app.setState(ApplicationState.SUBMITTED, state.getCurrentTime()); - state.itemUpdated(app); - } - else if (taskState == Status.ACTIVE) { - app.setState(ApplicationState.ACTIVE, state.getCurrentTime()); - state.itemUpdated(app); - } + } + + private ApplicationState getAppStateFromTaskState(int taskState) { + switch (taskState) { + case Status.SUBMITTING: + return ApplicationState.SUBMITTING; + case Status.SUBMITTED: + return ApplicationState.SUBMITTED; + case Status.ACTIVE: + return ApplicationState.ACTIVE; + case Status.STAGE_IN: + return ApplicationState.STAGE_IN; + case Status.STAGE_OUT: + return ApplicationState.STAGE_OUT; + default: + return ApplicationState.INITIALIZING; } } Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppEndProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppEndProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppEndProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -22,6 +22,8 @@ import org.apache.log4j.Level; import org.griphyn.vdl.karajan.monitor.SystemState; +import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; +import org.griphyn.vdl.karajan.monitor.items.ApplicationState; import org.griphyn.vdl.karajan.monitor.items.StatefulItem; import org.griphyn.vdl.karajan.monitor.items.StatefulItemClass; import org.griphyn.vdl.karajan.monitor.processors.SimpleParser; @@ -29,21 +31,23 @@ public class AppEndProcessor extends AbstractSwiftProcessor { public Level getSupportedLevel() { - return Level.DEBUG; + return Level.INFO; } @Override public String getMessageHeader() { - return "JOB_END"; + return "END_SUCCESS"; } public void processMessage(SystemState state, SimpleParser p, Object details) { try { - p.skip("jobid="); - String id = p.word(); + p.skip("thread="); + String threadid = p.word(); - StatefulItem app = state.getItemByID(id, - StatefulItemClass.APPLICATION); + StatefulItem thread = state.getItemByID(threadid, StatefulItemClass.BRIDGE); + ApplicationItem app = (ApplicationItem) thread.getParent(); + app.setState(ApplicationState.FINISHED_SUCCESSFULLY, state.getCurrentTime()); + state.itemUpdated(app); state.removeItem(app); state.getStats("apps").remove(); } Added: trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppFailureProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppFailureProcessor.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppFailureProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,58 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* + * Created on Jan 29, 2007 + */ +package org.griphyn.vdl.karajan.monitor.processors.swift; + +import org.apache.log4j.Level; +import org.griphyn.vdl.karajan.monitor.SystemState; +import org.griphyn.vdl.karajan.monitor.items.ApplicationItem; +import org.griphyn.vdl.karajan.monitor.items.ApplicationState; +import org.griphyn.vdl.karajan.monitor.items.StatefulItem; +import org.griphyn.vdl.karajan.monitor.items.StatefulItemClass; +import org.griphyn.vdl.karajan.monitor.processors.SimpleParser; + +public class AppFailureProcessor extends AbstractSwiftProcessor { + + public Level getSupportedLevel() { + return Level.DEBUG; + } + + @Override + public String getMessageHeader() { + return "END_FAILURE"; + } + + public void processMessage(SystemState state, SimpleParser p, Object details) { + try { + p.skip("thread="); + String threadid = p.word(); + + StatefulItem thread = state.getItemByID(threadid, StatefulItemClass.BRIDGE); + ApplicationItem app = (ApplicationItem) thread.getParent(); + app.setState(ApplicationState.FAILED, state.getCurrentTime()); + state.itemUpdated(app); + state.removeItem(app); + state.getStats("apps").remove(); + } + catch (Exception e) { + e.printStackTrace(); + } + } +} Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppStartProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppStartProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/AppStartProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -48,7 +48,7 @@ String args = ""; if (p.matchAndSkip("arguments=[")) { p.beginToken(); - p.markTo("]"); + p.markMatchedTo(']', '['); args = p.getToken(); } p.skip("host="); Added: trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/ConfigurationProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/ConfigurationProcessor.java (rev 0) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/ConfigurationProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -0,0 +1,65 @@ +/* + * Copyright 2012 University of Chicago + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* + * Created on Aug 28, 2008 + */ +package org.griphyn.vdl.karajan.monitor.processors.swift; + +import org.apache.log4j.Level; +import org.griphyn.vdl.karajan.Loader; +import org.griphyn.vdl.karajan.monitor.SystemState; +import org.griphyn.vdl.karajan.monitor.processors.AbstractMessageProcessor; +import org.griphyn.vdl.karajan.monitor.processors.ParsingException; +import org.griphyn.vdl.karajan.monitor.processors.SimpleParser; + +public class ConfigurationProcessor extends AbstractMessageProcessor { + + public Level getSupportedLevel() { + return Level.DEBUG; + } + + public Class getSupportedSource() { + return Loader.class; + } + + public void processMessage(SystemState state, Object message, Object details) { + String msg = String.valueOf(message); + SimpleParser p = new SimpleParser(msg); + try { + if (p.matchAndSkip("SWIFT_CONFIGURATION")) { + p.skipTo(": {"); + p.beginToken(); + p.markTo("}"); + String options = p.getToken(); + String[] els = options.split(",\\s*"); + for (String el : els) { + String[] kv = el.split("=", 2); + if (kv[0].equals("execution.retries")) { + state.setRetries(Integer.parseInt(kv[1])); + } + else if (kv[0].equals("replication.enabled")) { + state.setReplicationEnabled(Boolean.parseBoolean(kv[1])); + } + } + } + } + catch (ParsingException e) { + e.printStackTrace(); + } + } +} Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/JobProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/JobProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/JobProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -20,10 +20,15 @@ */ package org.griphyn.vdl.karajan.monitor.processors.swift; +import org.apache.log4j.Level; import org.griphyn.vdl.karajan.lib.Execute; import org.griphyn.vdl.karajan.monitor.processors.karajan.TaskProcessor; public class JobProcessor extends TaskProcessor { + + public Level getSupportedLevel() { + return Level.INFO; + } public Class getSupportedSource() { return Execute.class; Modified: trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java 2014-07-04 06:52:32 UTC (rev 7964) +++ trunk/src/org/griphyn/vdl/karajan/monitor/processors/swift/SummaryProcessor.java 2014-07-04 06:54:51 UTC (rev 7965) @@ -41,8 +41,30 @@ public void processMessage(SystemState state, Object message, Object details) { String msg = String.valueOf(message); if(msg.contains("CrtHeap")) { - return; + processJVMInfo(state, msg); } + else { + processTaskInfo(state, msg); + } + } + + private void processJVMInfo(SystemState state, String msg) { + String[] els = msg.split(",\\s"); + for (String el : els) { + String[] kv = el.split(": "); + if ("HeapMax".equals(kv[0])) { + state.setMaxHeap(Long.parseLong(kv[1])); + } + else if ("UsedHeap".equals(kv[0])) { + state.setUsedHeap(Long.parseLong(kv[1])); + } + else if ("JVMThreads".equals(kv[0])) { + state.setCurrentThreads(Integer.parseInt(kv[1])); + } + } + } + + private void processTaskInfo(SystemState state, String msg) { SummaryItem s = (SummaryItem) state.getItemByID(SummaryItem.ID, StatefulItemClass.WORKFLOW); String[] pairs = msg.split(" "); for (ApplicationState key : SummaryItem.STATES) { From hategan at ci.uchicago.edu Fri Jul 4 02:01:55 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:01:55 -0500 (CDT) Subject: [Swift-commit] r7967 - in trunk/resources/httpmonitor: . css images Message-ID: <20140704070155.E8D619D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 02:01:55 -0500 (Fri, 04 Jul 2014) New Revision: 7967 Added: trunk/resources/httpmonitor/css/jqpagination.css trunk/resources/httpmonitor/images/layout.png Modified: trunk/resources/httpmonitor/css/style.css trunk/resources/httpmonitor/index.html Log: http monitor updates (client side) Added: trunk/resources/httpmonitor/css/jqpagination.css =================================================================== --- trunk/resources/httpmonitor/css/jqpagination.css (rev 0) +++ trunk/resources/httpmonitor/css/jqpagination.css 2014-07-04 07:01:55 UTC (rev 7967) @@ -0,0 +1,86 @@ +.pagination { + display: inline-block; + border: 1px solid #CDCDCD; + border-radius: 3px; } + +.pagination a { + display: block; + float: left; + width: 20px; + height: 20px; + outline: none; + border-right: 1px solid #CDCDCD; + border-left: 1px solid #CDCDCD; + color: #555555; + vertical-align: middle; + text-align: center; + text-decoration: none; + font-weight: bold; + font-size: 16px; + font-family: Times, 'Times New Roman', Georgia, Palatino; + /* ATTN: need a better font stack */ + background-color: #f3f3f3; + background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #f3f3f3), color-stop(100%, lightgrey)); + background-image: -webkit-linear-gradient(#f3f3f3, lightgrey); + background-image: linear-gradient(#f3f3f3, lightgrey); } + .pagination a:hover, .pagination a:focus, .pagination a:active { + background-color: #cecece; + background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #e4e4e4), color-stop(100%, #cecece)); + background-image: -webkit-linear-gradient(#e4e4e4, #cecece); + background-image: linear-gradient(#e4e4e4, #cecece); } + .pagination a.disabled, .pagination a.disabled:hover, .pagination a.disabled:focus, .pagination a.disabled:active { + background-color: #f3f3f3; + background-image: -webkit-gradient(linear, left top, left bottom, color-stop(0%, #f3f3f3), color-stop(100%, lightgrey)); + background-image: -webkit-linear-gradient(#f3f3f3, lightgrey); + background-image: linear-gradient(#f3f3f3, lightgrey); + color: #A8A8A8; + cursor: default; } + +.pagination a:first-child { + border: none; + border-radius: 2px 0 0 2px; } + +.pagination a:last-child { + border: none; + border-radius: 0 2px 2px 0; } + +.pagination input { + float: left; + margin: 0; + padding: 0; + width: 120px; + height: 20px; + outline: none; + border: none; + vertical-align: middle; + text-align: center; } + +/* gigantic class for demo purposes */ +.gigantic.pagination { + margin: 30px 0; } + +.gigantic.pagination a { + height: 60px; + width: 60px; + font-size: 50px; + line-height: 50px; } + +.gigantic.pagination input { + width: 300px; + height: 60px; + font-size: 30px; } + +/* log element for demo purposes */ +.log { + display: none; + background-color: #EDEDED; + border: 1px solid #B4B4B4; + height: 300px; + width: 524px; + overflow: auto; + margin-left: 0; + list-style: none; + padding: 10px; } + .log li { + margin-top: 0; + margin-bottom: 5px; } Modified: trunk/resources/httpmonitor/css/style.css =================================================================== --- trunk/resources/httpmonitor/css/style.css 2014-07-04 07:01:07 UTC (rev 7966) +++ trunk/resources/httpmonitor/css/style.css 2014-07-04 07:01:55 UTC (rev 7967) @@ -11,6 +11,22 @@ width: 100%; } +#summary-container { +} + +#summary-container, #heap-container { + width: 600px; + margin-left: auto; + margin-right: auto; +} + +#pbi-main { + width: 95%; + margin-left: auto; + margin-right: auto; + margin-bottom: 38px; +} + .bordered { margin: 4px; padding: 4px; @@ -18,11 +34,11 @@ .mlabel { position: relative; - top: 14px; + top: -14px; left: 16px; padding-left: 9px; padding-right: 4px; - width: 80px; + width: 100px; } #appSummary .label { @@ -151,4 +167,89 @@ margin: 2px; padding: 4px; background-color: white; +} + + +th { + font-size: 12px; + font-weight: bold; +} + +th:first-child { + border-left: none; +} + +th { + /*border-left: thin solid black; + border-bottom: thin solid black;*/ + padding-left: 8px; + padding-right: 8px; + padding-bottom: 4px; + margin: 0px; +} + +td { + font-size: 12px; + padding: 4px; + padding-left: 8px; + padding-right: 8px; +} + +.alt-shaded tr:nth-child(odd) { + background-color: #f0f0f0; +} + +td.numeric { + text-align: right; + padding-right: 8px; +} + +table.collapsed-border { + border-collapse: collapse; +} + +.toolbar li { + display: inline; +} + +.paging { + text-align: center; +} + +#browser h1 { + font-size: 14pt; + width: 99%; + background-color: #f0f0f0; + padding: 4px 0px 4px 8px; + margin-right: 16px; + margin-left: 0px; +} + +#browser em { + font-weight: bold; +} + +.plot-medium { + width: 400px; + height: 200px; +} + +.plot-small { + width: 240px; + height: 120px; +} + + +.plot-strip { + width: 400px; + height: 16px; +} + + +.centered { + text-align: center; +} + +.float-left { + float: left; } \ No newline at end of file Added: trunk/resources/httpmonitor/images/layout.png =================================================================== (Binary files differ) Property changes on: trunk/resources/httpmonitor/images/layout.png ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Modified: trunk/resources/httpmonitor/index.html =================================================================== --- trunk/resources/httpmonitor/index.html 2014-07-04 07:01:07 UTC (rev 7966) +++ trunk/resources/httpmonitor/index.html 2014-07-04 07:01:55 UTC (rev 7967) @@ -1,3 +1,4 @@ + Swift System Monitor @@ -5,82 +6,128 @@ + + + + + + + + + + + + +
-
    + -
    -
    App Summary
    -
    +
    +
    +
    +
    App Summary
    - +
    -
    +
    Heap:
    -
    - + -
    -
    - - + + + + + + + + \ No newline at end of file From swift at ci.uchicago.edu Fri Jul 4 02:05:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:05:03 -0500 (CDT) Subject: [Swift-commit] cog r3996 Message-ID: <20140704070503.C072C8D000A5@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r3996 | hategan | 2014-07-04 02:04:40 -0500 (Fri, 04 Jul 2014) | 1 line no need to keep stack frame count if starting a new thread ------------------------------------------------------------------------ Index: modules/karajan/src/org/globus/cog/karajan/compiled/nodes/UParallel.java =================================================================== --- modules/karajan/src/org/globus/cog/karajan/compiled/nodes/UParallel.java (revision 3995) +++ modules/karajan/src/org/globus/cog/karajan/compiled/nodes/UParallel.java (working copy) @@ -57,17 +57,12 @@ int state = thr.checkSliceAndPopState(); Stack stack = thr.getStack(); ThreadSetFixed ts = (ThreadSetFixed) thr.popState(); - int fc = thr.popIntState(); try { switch (state) { case 0: - fc = stack.frameCount(); - state++; - case 1: int ec = childCount(); final ThreadSetFixed tsf = new ThreadSetFixed(ec); ts = tsf; - final int fcf = fc; for (int i = 0; i < ec; i++) { final int fi = i; LWThread ct = thr.fork(i, new KRunnable() { @@ -78,14 +73,10 @@ tsf.threadDone(thr, null); } catch (ExecutionException e) { - Stack stack = thr.getStack(); - stack.dropToFrame(fcf); tsf.threadDone(thr, e); tsf.abortAll(); } catch (RuntimeException e) { - Stack stack = thr.getStack(); - stack.dropToFrame(fcf); tsf.threadDone(thr, new ExecutionException(UParallel.this, e)); tsf.abortAll(); } @@ -97,12 +88,11 @@ } ts.startAll(); state++; - case 2: + case 1: ts.waitFor(); } } catch (Yield y) { - y.getState().push(fc); y.getState().push(ts); y.getState().push(state); throw y; From swift at ci.uchicago.edu Fri Jul 4 02:10:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:10:03 -0500 (CDT) Subject: [Swift-commit] cog r4001 Message-ID: <20140704071003.E5DEE8D000A5@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r4001 | hategan | 2014-07-04 02:09:30 -0500 (Fri, 04 Jul 2014) | 1 line added ability to remove service attributes ------------------------------------------------------------------------ Index: modules/abstraction-common/src/org/globus/cog/abstraction/impl/common/task/ServiceImpl.java =================================================================== --- modules/abstraction-common/src/org/globus/cog/abstraction/impl/common/task/ServiceImpl.java (revision 4000) +++ modules/abstraction-common/src/org/globus/cog/abstraction/impl/common/task/ServiceImpl.java (working copy) @@ -172,6 +172,13 @@ } } + @Override + public void removeAttribute(String name) { + if (attributes != null) { + attributes.remove(name); + } + } + public String toString() { return serviceContact + "(" + this.provider + ")"; } Index: modules/abstraction-common/src/org/globus/cog/abstraction/interfaces/Service.java =================================================================== --- modules/abstraction-common/src/org/globus/cog/abstraction/interfaces/Service.java (revision 4000) +++ modules/abstraction-common/src/org/globus/cog/abstraction/interfaces/Service.java (working copy) @@ -162,4 +162,6 @@ public Enumeration getAllAttributes(); public Collection getAttributeNames(); + + public void removeAttribute(String name); } From swift at ci.uchicago.edu Fri Jul 4 02:15:04 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:15:04 -0500 (CDT) Subject: [Swift-commit] cog r4004 Message-ID: <20140704071504.9AB618D000A5@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r4004 | hategan | 2014-07-04 02:14:31 -0500 (Fri, 04 Jul 2014) | 1 line only output diagnostic messages if at least one byte was sent/received ------------------------------------------------------------------------ Index: modules/provider-coaster/src/org/globus/cog/coaster/channels/PerformanceDiagnosticOutputStream.java =================================================================== --- modules/provider-coaster/src/org/globus/cog/coaster/channels/PerformanceDiagnosticOutputStream.java (revision 4003) +++ modules/provider-coaster/src/org/globus/cog/coaster/channels/PerformanceDiagnosticOutputStream.java (working copy) @@ -28,7 +28,7 @@ Timer.every(1000, new Runnable() { public void run() { count += 1; - if (count % INTERVAL == 0) { + if (count % INTERVAL == 0 && getTotal() > 0) { String s; logger.info(s = "[OUT] Total transferred: " + units(getTotal()) + "B, current rate: " Index: modules/provider-coaster/src/org/globus/cog/coaster/channels/PerformanceDiagnosticInputStream.java =================================================================== --- modules/provider-coaster/src/org/globus/cog/coaster/channels/PerformanceDiagnosticInputStream.java (revision 4003) +++ modules/provider-coaster/src/org/globus/cog/coaster/channels/PerformanceDiagnosticInputStream.java (working copy) @@ -29,16 +29,11 @@ Timer.every(1000, new Runnable() { public void run() { count += 1; - if (count % INTERVAL == 0) { + if (count % INTERVAL == 0 && getTotal() > 0) { String s; logger.info(s = "[IN] Total transferred: " + units(getTotal()) + "B, current rate: " + units(getCurrentRate()) + "B/s, average rate: " + units(getAverageRate()) + "B/s"); - logger.info(s = "[MEM] Heap total: " - + units(Runtime.getRuntime().totalMemory()) - + "B, Heap used: " - + units(Runtime.getRuntime().totalMemory() - - Runtime.getRuntime().freeMemory()) + "B"); } last = bytes; } From swift at ci.uchicago.edu Fri Jul 4 02:20:04 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:20:04 -0500 (CDT) Subject: [Swift-commit] cog r4008 Message-ID: <20140704072004.AB9818D000A5@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r4008 | hategan | 2014-07-04 02:19:23 -0500 (Fri, 04 Jul 2014) | 1 line removed logging of job details; it is now done in swift for all providers ------------------------------------------------------------------------ Index: modules/provider-local/src/org/globus/cog/abstraction/impl/execution/local/JobSubmissionTaskHandler.java =================================================================== --- modules/provider-local/src/org/globus/cog/abstraction/impl/execution/local/JobSubmissionTaskHandler.java (revision 4007) +++ modules/provider-local/src/org/globus/cog/abstraction/impl/execution/local/JobSubmissionTaskHandler.java (working copy) @@ -124,11 +124,11 @@ if (logger.isDebugEnabled()) { logger.debug("Submitting task " + task); } - else if (logger.isInfoEnabled()) { + else if (logger.isDebugEnabled()) { Specification spec = task.getSpecification(); if (spec instanceof JobSpecification) { JobSpecification jobspec = (JobSpecification) spec; - logger.info("Submit: " + + logger.debug("Submit: " + "in: " + jobspec.getDirectory() + " command: " + jobspec.getExecutable() + " " + jobspec.getArguments()); From swift at ci.uchicago.edu Fri Jul 4 02:25:04 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:25:04 -0500 (CDT) Subject: [Swift-commit] cog r4013 Message-ID: <20140704072505.1C7CD8D000A5@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r4013 | hategan | 2014-07-04 02:23:20 -0500 (Fri, 04 Jul 2014) | 1 line not used ------------------------------------------------------------------------ Index: modules/examples/.classpath =================================================================== --- modules/examples/.classpath (revision 4012) +++ modules/examples/.classpath (working copy) @@ -1,13 +0,0 @@ - - - - - - - - - - - - - Index: modules/examples/project.properties =================================================================== --- modules/examples/project.properties (revision 4012) +++ modules/examples/project.properties (working copy) @@ -1,6 +0,0 @@ -module.name = examples -long.name = Various examples for the Java COG Kit -version = 2.2 -project = Java CoG Kit -lib.deps = - -debug = true Index: modules/examples/launchers.xml =================================================================== --- modules/examples/launchers.xml (revision 4012) +++ modules/examples/launchers.xml (working copy) @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Index: modules/examples/.project =================================================================== --- modules/examples/.project (revision 4012) +++ modules/examples/.project (working copy) @@ -1,20 +0,0 @@ - - - examples - - - abstraction-common - grapheditor - jglobus - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.core.javanature - - Index: modules/examples/dependencies.xml =================================================================== --- modules/examples/dependencies.xml (revision 4012) +++ modules/examples/dependencies.xml (working copy) @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - Index: modules/examples/CHANGES.txt =================================================================== --- modules/examples/CHANGES.txt (revision 4012) +++ modules/examples/CHANGES.txt (working copy) @@ -1,89 +0,0 @@ -(08/29/2006) v2.2 - -*** Merged most things (besides DAG examples) into - abstraction-common - -(02/08/2006) - -*** Added a -stdin option for cog-job-submit. - -(08/23/2005) - -*** Added org.globus.cog.abstractions.examples.etc - package to display the client-side environment. Useful for - debugging purposes. Fixed bug: - http://www.cogkit.org/bugzilla/show_bug.cgi?id=161 - -(08/11/2005) - -*** Added support for the -specification option in the - JobSubmission example - -(07/01/2005) - -*** Added the RemoteCompileExecute.java example in - org.globus.cog.abstractions.examples.taskgraph package - -(06/08/2005) - -*** Updated the JobSubmission example to accomodate - job-manager abstractions - -(06/02/2005) - -*** Implemented the CheckpointStatus.java example in the - org.globus.cog.abstractions.examples.xml package and - added the corresponding cog-checkpoint-status launcher - --------------------------------2.1---------------------------------- - -(05/29/2005) - -*** Added the cog-checkpoint-submit and the cog-task2xml - launchers - -(05/29/2005) - -*** improved the task examples in the - org.globus.cog.abstractions.examples.execution package - to handle "attributes" "checkpoint" and "environment" - options - -(05/29/2005) - -*** improved the task checkpointing examples in the - org.globus.cog.abstractions.examples.xml package - -(04/18/2005) - -*** renamed the module from "core-examples" to - "abstraction-examples" - --------------------------------2.0---------------------------------- - -(03/24/2005) - -*** removed redundant files in the - src/org/globus/cog/core/examples/file directory. Instead - added a single FileOperation.java example that provides a - generic example for all file operations. - -(03/24/2005) - -*** removed the redundant launchers for file operation tests - and examples. Instead added a single launcher for the file - operation example. - -(03/17/2005) - -*** moved the cog-job-submit and cog-file-transfer launachers - from the dist/cog-/bin/example directory to the - dist/cog-/bin directory. - -(03/17/2005) - -*** added the cogrun launcher in the dist/cog-/bin - directory. - --------------------------------1.0---------------------------------- - Index: modules/examples/meta/cog-job-submit/description.txt =================================================================== --- modules/examples/meta/cog-job-submit/description.txt (revision 4012) +++ modules/examples/meta/cog-job-submit/description.txt (working copy) @@ -1,9 +0,0 @@ -This command submits a user-specified executable for remote -execution. The user can specify the service contact for the job -manager, the provider, the executable, and the optional -arguments. Several other optional parameters control the behavior -of the execution. The user can execute this job as a batch job, -whereby the job status will not be updated on the client side. The -user can also redirect the job output and error to a user-supplied -file, either on the remote machine or on the local machine. It also allows -long-running jobs to be checkpointed to a local file. Index: modules/examples/meta/cog-job-submit/usage.txt =================================================================== --- modules/examples/meta/cog-job-submit/usage.txt (revision 4012) +++ modules/examples/meta/cog-job-submit/usage.txt (working copy) @@ -1,57 +0,0 @@ -Usage: - cog-job-submit - [(-name | -n) ] - Task name - - (-service-contact | -s) - Service contact - - [(-job-manager | -jm) ] - Execution JobManager (fork, pbs, etc) - - [(-provider | -p) ] - Provider; available providers: [gt2ft, gsiftp, file, gt4.0.0, - gt3.0.2, ssh, gt4ft, gridftp, local, gsiftp-old, http, gt3.2.1, - gt2, gt3.2.0, gridftp-old, ftp, webdav] - - (-executable | -e) - Executable - - [(-arguments | -args) ] - Arguments. If more than one, use quotes - - [(-environment | -env) ] - Environment variables for the remote execution environment, - specified as "name=value[,name=value]" - - [(-directory | -d) ] - Target directory - - [(-batch | -b)] - If present, the job is run in batch mode - - [(-redirected | -r)] - If present, the arguments to -stdout and -stderr refer to local - files - - [-stdout ] - Indicates a file where the standard output of the job should be - redirected - - [-stderr ] - Indicates a file where the standard error of the job should be - redirected - - [(-attributes | -a) ] - Additional task specification attributes. Attributes can be - specified as "name=value[,name=value]" - - [(-checkpoint | -c) ] - Checkpoint file name. The task will be checkpointed to this file - once submitted - - [(-verbose | -v)] - If enabled, display information about what is being done - - [(-help | -h)] - Display usage Index: modules/examples/meta/description.txt =================================================================== --- modules/examples/meta/description.txt (revision 4012) +++ modules/examples/meta/description.txt (working copy) @@ -1,11 +0,0 @@ -The abstraction-example module contains the code base for several command -line clients and working examples showcasing various important concepts of -the abstraction framework. Contained examples demonstrate the following -features of the Java Cog Kit: - -- How to submit jobs on remote Grid resources -- How to transfer files betwen two Grid resources -- How to invoke file operations on remote Grid resources -- How to checkpoint and resubmit task executions -- How to execute hierarchical taskgraphs, queues, and sets -- How to visualize hierarchical taskgraph executions Index: modules/examples/meta/cog-file-operation/description.txt =================================================================== --- modules/examples/meta/cog-file-operation/description.txt (revision 4012) +++ modules/examples/meta/cog-file-operation/description.txt (working copy) @@ -1,28 +0,0 @@ -This command initiates a connection with a remote file server and -allows the user to invoke operations on files hosted on that -server. On execution, this commands enters a shell-based mode allowing the -user to invoke file operation commands. The following commands are -supported: - -Java CoG Kit File Operation commands ------------------------------------- -Please Note: Commands are case sensitive and all arguments are -required - -1. stop -2. ls -3. ls -4. pwd -5. cd -6. mkdir -7. rmdir -8. rmfile -9. isDirectory -10. exists -11. getfile -12. putfile -13. getdir -14. putfile -15. rename -16. chmod Index: modules/examples/meta/cog-file-operation/usage.txt =================================================================== --- modules/examples/meta/cog-file-operation/usage.txt (revision 4012) +++ modules/examples/meta/cog-file-operation/usage.txt (working copy) @@ -1,15 +0,0 @@ -Usage: - cog-file-operation - (-service-contact | -s) - Service contact - - [(-provider | -p) ] - Provider; available providers: [gt2ft, gsiftp, file, gt4.0.0, - gt3.0.2, ssh, gt4ft, gridftp, local, gsiftp-old, http, gt3.2.1, - gt2, gt3.2.0, gridftp-old, ftp, webdav] - - [(-verbose | -v)] - If enabled, display information about what is being done - - [(-help | -h)] - Display usage Index: modules/examples/meta/cog-file-transfer/description.txt =================================================================== --- modules/examples/meta/cog-file-transfer/description.txt (revision 4012) +++ modules/examples/meta/cog-file-transfer/description.txt (working copy) @@ -1,3 +0,0 @@ -This command transfers a file hosted on one file server to another -file server. If both the file servers are GridFTP servers, then -the files can be transfered in third party mode. Index: modules/examples/meta/cog-file-transfer/usage.txt =================================================================== --- modules/examples/meta/cog-file-transfer/usage.txt (revision 4012) +++ modules/examples/meta/cog-file-transfer/usage.txt (working copy) @@ -1,15 +0,0 @@ -Usage: - cog-file-transfer - (-source-uri | -s) - Source URI: ://[:port]// - - (-destination-uri | -d) - Destination URI: - ://[:port]// - - [(-thirdparty | -t)] - If present, performs a third party file transfer. Valid only - between two GridFTP resources - - [(-help | -h)] - Display usage Index: modules/examples/meta/cog-task2xml/description.txt =================================================================== --- modules/examples/meta/cog-task2xml/description.txt (revision 4012) +++ modules/examples/meta/cog-task2xml/description.txt (working copy) @@ -1,9 +0,0 @@ -This command simply translates the given task into an XML format -without actually submitting the task. -The user can specify the service contact for the job -manager, the provider, the executable, and the optional -arguments. Several other optional parameters control the behavior -of the execution. The user can execute this job as a batch job, -whereby the job status will not be updated on the client side. The -user can also redirect the job output and error to a user-supplied -file, either on the remote machine or on the local machine. Index: modules/examples/meta/cog-task2xml/usage.txt =================================================================== --- modules/examples/meta/cog-task2xml/usage.txt (revision 4012) +++ modules/examples/meta/cog-task2xml/usage.txt (working copy) @@ -1,56 +0,0 @@ -Usage: - cog-task2xml - (-checkpoint | -c) - Checkpoint file name. The task will be checkpointed to this file - - [(-name | -n) ] - Task name - - (-service-contact | -s) - Service contact - - [(-job-manager | -jm) ] - Execution JobManager (fork, pbs, etc) - - [(-provider | -p) ] - Provider; available providers: [gt2ft, gsiftp, file, gt4.0.0, - gt3.0.2, ssh, gt4ft, gridftp, local, gsiftp-old, http, gt3.2.1, - gt2, gt3.2.0, gridftp-old, ftp, webdav] - - (-executable | -e) - Executable - - [(-arguments | -args) ] - Arguments. If more than one, use quotes - - [(-environment | -env) ] - Environment variables for the remote execution environment, - specified as "name=value[,name=value]" - - [(-directory | -d) ] - Target directory - - [(-batch | -b)] - If present, the job is run in batch mode - - [(-redirected | -r)] - If present, the arguments to -stdout and -stderr refer to local - files - - [-stdout ] - Indicates a file where the standard output of the job should be - redirected - - [-stderr ] - Indicates a file where the standard error of the job should be - redirected - - [(-attributes | -a) ] - Additional task specification attributes. Attributes can be - specified as "name=value[,name=value]" - - [(-verbose | -v)] - If enabled, display information about what is being done - - [(-help | -h)] - Display usage Index: modules/examples/meta/cogrun/description.txt =================================================================== --- modules/examples/meta/cogrun/description.txt (revision 4012) +++ modules/examples/meta/cogrun/description.txt (working copy) @@ -1,8 +0,0 @@ -This command submits a user-specified executable for remote -execution. The user can specify the service contact for the job -manager, the provider, the executable, and the optional -arguments. Several other optional parameters control the behavior -of the execution. The user can execute this job as a batch job, -whereby the job status will not be updated on the client side. The -user can also redirect the job output and error to a user-supplied -file, either on the remote machine or on the local machine. Index: modules/examples/meta/cogrun/usage.txt =================================================================== --- modules/examples/meta/cogrun/usage.txt (revision 4012) +++ modules/examples/meta/cogrun/usage.txt (working copy) @@ -1,57 +0,0 @@ -Usage: - cogrun - [(-name | -n) ] - Task name - - (-service-contact | -s) - Service contact - - [(-job-manager | -jm) ] - Execution JobManager (fork, pbs, etc) - - [(-provider | -p) ] - Provider; available providers: [gt2ft, gsiftp, file, gt4.0.0, - gt3.0.2, ssh, gt4ft, gridftp, local, gsiftp-old, http, gt3.2.1, - gt2, gt3.2.0, gridftp-old, ftp, webdav] - - (-executable | -e) - Executable - - [(-arguments | -args) ] - Arguments. If more than one, use quotes - - [(-environment | -env) ] - Environment variables for the remote execution environment, - specified as "name=value[,name=value]" - - [(-directory | -d) ] - Target directory - - [(-batch | -b)] - If present, the job is run in batch mode - - [(-redirected | -r)] - If present, the arguments to -stdout and -stderr refer to local - files - - [-stdout ] - Indicates a file where the standard output of the job should be - redirected - - [-stderr ] - Indicates a file where the standard error of the job should be - redirected - - [(-attributes | -a) ] - Additional task specification attributes. Attributes can be - specified as "name=value[,name=value]" - - [(-checkpoint | -c) ] - Checkpoint file name. The task will be checkpointed to this file - once submitted - - [(-verbose | -v)] - If enabled, display information about what is being done - - [(-help | -h)] - Display usage Index: modules/examples/meta/cog-checkpoint-status/description.txt =================================================================== --- modules/examples/meta/cog-checkpoint-status/description.txt (revision 4012) +++ modules/examples/meta/cog-checkpoint-status/description.txt (working copy) @@ -1,6 +0,0 @@ -This command allows the user to check the status of a long running -task. The task is represented by the checkpoint file (created by -the cogrun, cog-job-submit, or cog-task2xml commands). Upon -submission, the client re-connects to the remote execution service and -retreives the latest execution status. - Index: modules/examples/meta/cog-checkpoint-status/usage.txt =================================================================== --- modules/examples/meta/cog-checkpoint-status/usage.txt (revision 4012) +++ modules/examples/meta/cog-checkpoint-status/usage.txt (working copy) @@ -1,10 +0,0 @@ -Usage: - cog-checkpoint-status - (-checkpoint | -c) - Input checkpoint file - - [(-verbose | -v)] - If enabled, display information about what is being done - - [(-help | -h)] - Display usage Index: modules/examples/meta/cog-checkpoint-submit/description.txt =================================================================== --- modules/examples/meta/cog-checkpoint-submit/description.txt (revision 4012) +++ modules/examples/meta/cog-checkpoint-submit/description.txt (working copy) @@ -1,4 +0,0 @@ -This command allows the user to submit a checkpoint file (created by -the cogrun, cog-job-submit, or cog-task2xml commands). Upon -submission, the client re-connects to the remote execution service and -actively monitors the status updates. Index: modules/examples/meta/cog-checkpoint-submit/usage.txt =================================================================== --- modules/examples/meta/cog-checkpoint-submit/usage.txt (revision 4012) +++ modules/examples/meta/cog-checkpoint-submit/usage.txt (working copy) @@ -1,10 +0,0 @@ -Usage: - cog-checkpoint-submit - (-checkpoint | -c) - Input checkpoint file - - [(-verbose | -v)] - If enabled, display information about what is being done - - [(-help | -h)] - Display usage Index: modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/RemoteCompileExecute.java =================================================================== --- modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/RemoteCompileExecute.java (revision 4012) +++ modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/RemoteCompileExecute.java (working copy) @@ -1,303 +0,0 @@ -// ---------------------------------------------------------------------- -//This code is developed as part of the Java CoG Kit project -//The terms of the license can be found at http://www.cogkit.org/license -//This message may not be removed or altered. -//---------------------------------------------------------------------- - -package org.globus.cog.abstraction.examples.taskgraph; - -import java.io.File; - -import org.apache.log4j.Logger; -import org.globus.cog.abstraction.impl.common.AbstractionFactory; -import org.globus.cog.abstraction.impl.common.AbstractionProperties; -import org.globus.cog.abstraction.impl.common.ProviderMethodException; -import org.globus.cog.abstraction.impl.common.StatusEvent; -import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl; -import org.globus.cog.abstraction.impl.common.task.IllegalSpecException; -import org.globus.cog.abstraction.impl.common.task.InvalidProviderException; -import org.globus.cog.abstraction.impl.common.task.InvalidSecurityContextException; -import org.globus.cog.abstraction.impl.common.task.InvalidServiceContactException; -import org.globus.cog.abstraction.impl.common.task.JobSpecificationImpl; -import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl; -import org.globus.cog.abstraction.impl.common.task.TaskImpl; -import org.globus.cog.abstraction.impl.common.task.TaskSubmissionException; -import org.globus.cog.abstraction.impl.common.taskgraph.TaskGraphHandlerImpl; -import org.globus.cog.abstraction.impl.common.taskgraph.TaskGraphImpl; -import org.globus.cog.abstraction.interfaces.ExecutableObject; -import org.globus.cog.abstraction.interfaces.ExecutionService; -import org.globus.cog.abstraction.interfaces.JobSpecification; -import org.globus.cog.abstraction.interfaces.SecurityContext; -import org.globus.cog.abstraction.interfaces.ServiceContact; -import org.globus.cog.abstraction.interfaces.Status; -import org.globus.cog.abstraction.interfaces.StatusListener; -import org.globus.cog.abstraction.interfaces.Task; -import org.globus.cog.abstraction.interfaces.TaskGraph; -import org.globus.cog.abstraction.interfaces.TaskGraphHandler; -import org.globus.cog.abstraction.tools.transfer.FileTransfer; -import org.globus.cog.util.ArgumentParser; -import org.globus.cog.util.ArgumentParserException; - -/* - * This example demonstrates an execution flow where a local java file needs to - * be compiled and executed remotely. Then it transfers the stdout of the remote - * execution back to the local machine. - */ -public class RemoteCompileExecute implements StatusListener { - static Logger logger = Logger.getLogger(RemoteCompileExecute.class - .getName()); - private String serviceContact = null; - private String executionProvider = null; - private String transferProvider = null; - private String sourceFile = null; - private String normalizedFileName = null; - private String javaHome = null; - private String classpath = null; - private TaskGraph tg = null; - - public static void main(String args[]) { - ArgumentParser ap = new ArgumentParser(); - ap.setExecutableName("cog-compile-execute"); - ap.addOption("src", "Name of the Java source file", "filename", - ArgumentParser.NORMAL); - ap.addAlias("src", "s"); - ap.addOption("javaHome", - "Path of the Java environment on the remote machine", "path", - ArgumentParser.NORMAL); - ap.addAlias("javaHome", "j"); - ap.addOption("classpath", - "Path of the Java classpath on the remote machine", "path", - ArgumentParser.OPTIONAL); - ap.addAlias("classpath", "cp"); - ap.addOption("execution-provider", - "Execution provider; available providers: " - + AbstractionProperties.getProviders().toString(), - "provider", ArgumentParser.NORMAL); - ap.addAlias("execution-provider", "ep"); - ap.addOption("transfer-provider", - "Transfer provider; available providers: " - + AbstractionProperties.getProviders().toString(), - "provider", ArgumentParser.NORMAL); - ap.addAlias("transfer-provider", "tp"); - ap.addOption("service-contact", "Service contact for the remote host", - "host", ArgumentParser.NORMAL); - ap.addAlias("service-contact", "sc"); - ap.addFlag("verbose", - "If enabled, display information about what is being done"); - ap.addAlias("verbose", "v"); - ap.addFlag("help", "Display usage"); - ap.addAlias("help", "h"); - try { - ap.parse(args); - if (ap.isPresent("help")) { - ap.usage(); - } else { - ap.checkMandatory(); - try { - RemoteCompileExecute rce = new RemoteCompileExecute(); - rce.setServiceContact(ap.getStringValue("service-contact")); - rce.setExecutionProvider(ap - .getStringValue("execution-provider")); - rce.setTransferProvider(ap - .getStringValue("transfer-provider")); - rce.setSourceFile(ap.getStringValue("src")); - rce.setJavaHome(ap.getStringValue("javaHome")); - rce.setClasspath(ap.getStringValue("classpath", null)); - rce.prepareTaskGraph(); - rce.submit(); - } catch (Exception e) { - logger.error("Exception in main", e); - } - } - } catch (ArgumentParserException e) { - System.err.println("Error parsing arguments: " + e.getMessage()); - ap.usage(); - } - } - - private void prepareTaskGraph() throws Exception { - this.tg = new TaskGraphImpl(); - tg.setName("Workflow"); - Task transferTask = prepareFileTransferTask(); - Task compileTask = prepareCompileTask(); - Task executeTask = prepareExecutionTask(); - tg.add(transferTask); - tg.add(compileTask); - tg.add(executeTask); - - tg.addDependency(transferTask, compileTask); - tg.addDependency(compileTask, executeTask); - tg.addStatusListener(this); - } - - private void submit() { - TaskGraphHandler handler = new TaskGraphHandlerImpl(); - try { - handler.submit(this.tg); - logger.debug("TaskGraph submitted"); - } catch (InvalidSecurityContextException ise) { - logger.error("Security Exception"); - ise.printStackTrace(); - System.exit(1); - } catch (TaskSubmissionException tse) { - logger.error("TaskSubmission Exception"); - tse.printStackTrace(); - System.exit(1); - } catch (IllegalSpecException ispe) { - logger.error("Specification Exception"); - ispe.printStackTrace(); - System.exit(1); - } catch (InvalidServiceContactException isce) { - logger.error("Service Contact Exception"); - isce.printStackTrace(); - System.exit(1); - } - } - - private Task prepareFileTransferTask() throws Exception { - FileTransfer fileTransfer = new FileTransfer("Transfer_Task", "file://" - + this.sourceFile, this.transferProvider + "://" - + this.serviceContact + "///tmp/" + normalizedFileName - + ".java"); - fileTransfer.prepareTask(); - Task task = fileTransfer.getFileTransferTask(); - task.removeStatusListener(fileTransfer); - task.addStatusListener(this); - return task; - } - - private Task prepareCompileTask() throws InvalidProviderException, - ProviderMethodException { - Task task = new TaskImpl("Compile_Task", Task.JOB_SUBMISSION); - JobSpecification spec = new JobSpecificationImpl(); - - spec.setExecutable(this.javaHome + "/bin/javac"); - if (this.classpath != null) { - spec.addArgument(" -classpath"); - spec.addArgument(this.classpath); - } - spec.addArgument("/tmp/" + this.normalizedFileName + ".java"); - spec.setRedirected(true); - task.setSpecification(spec); - - ExecutionService service = new ExecutionServiceImpl(); - service.setProvider(this.executionProvider.toLowerCase()); - - SecurityContext securityContext = AbstractionFactory - .newSecurityContext(this.executionProvider.toLowerCase()); - securityContext.setCredentials(null); - service.setSecurityContext(securityContext); - - ServiceContact sc = new ServiceContactImpl(this.serviceContact); - service.setServiceContact(sc); - task.addService(service); - - task.addStatusListener(this); - return task; - } - - private Task prepareExecutionTask() throws InvalidProviderException, - ProviderMethodException { - Task task = new TaskImpl("Execution_Task", Task.JOB_SUBMISSION); - - JobSpecification spec = new JobSpecificationImpl(); - - spec.setExecutable(this.javaHome + "/bin/java"); - if (this.classpath == null) { - spec.addArgument(" -classpath"); - spec.addArgument("/tmp"); - } else { - spec.addArgument(" -classpath"); - spec.addArgument("/tmp" + File.pathSeparator + this.classpath); - } - spec.addArgument(normalizedFileName); - spec.setRedirected(true); - task.setSpecification(spec); - - ExecutionService service = new ExecutionServiceImpl(); - service.setProvider(this.executionProvider.toLowerCase()); - - SecurityContext securityContext = AbstractionFactory - .newSecurityContext(this.executionProvider.toLowerCase()); - securityContext.setCredentials(null); - service.setSecurityContext(securityContext); - - ServiceContact sc = new ServiceContactImpl(this.serviceContact); - service.setServiceContact(sc); - task.addService(service); - - task.addStatusListener(this); - return task; - } - - public String getExecutionProvider() { - return executionProvider; - } - - public void setExecutionProvider(String provider) { - this.executionProvider = provider; - } - - public String getSourceFile() { - return sourceFile; - } - - public void setSourceFile(String sourceFile) { - this.sourceFile = sourceFile; - String filename = this.sourceFile.substring(this.sourceFile - .lastIndexOf("/")); - filename = filename.substring(1, filename.lastIndexOf(".java")); - this.normalizedFileName = filename; - logger.debug("Normalized file " + this.normalizedFileName); - } - - public String getServiceContact() { - return serviceContact; - } - - public void setServiceContact(String serviceContact) { - this.serviceContact = serviceContact; - } - - public String getJavaHome() { - return javaHome; - } - - public void setJavaHome(String javaHome) { - this.javaHome = javaHome; - } - - public void statusChanged(StatusEvent event) { - ExecutableObject eo = event.getSource(); - Status status = event.getStatus(); - if (eo.getObjectType() == ExecutableObject.TASK) { - logger.debug("Status of " + eo.getName() + " changed to: " - + status.getStatusString()); - if (status.getStatusCode() == Status.COMPLETED - && ((Task) eo).getStdOutput() != null) { - System.out.println("Output = " + ((Task) eo).getStdOutput()); - } - } else { - if (status.getStatusCode() == Status.COMPLETED - || status.getStatusCode() == Status.FAILED) { - System.exit(0); - } - } - } - - public String getClasspath() { - return classpath; - } - - public void setClasspath(String classpath) { - this.classpath = classpath; - } - - public String getTransferProvider() { - return transferProvider; - } - - public void setTransferProvider(String transferProvider) { - this.transferProvider = transferProvider; - } -} \ No newline at end of file Index: modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/HierarchicalDAGVisualizer.java =================================================================== --- modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/HierarchicalDAGVisualizer.java (revision 4012) +++ modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/HierarchicalDAGVisualizer.java (working copy) @@ -1,324 +0,0 @@ -// ---------------------------------------------------------------------- -// This code is developed as part of the Java CoG Kit project -// The terms of the license can be found at http://www.cogkit.org/license -// This message may not be removed or altered. -// ---------------------------------------------------------------------- - -package org.globus.cog.abstraction.examples.taskgraph; - -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.util.Enumeration; -import java.util.Hashtable; - -import org.globus.cog.abstraction.impl.common.StatusEvent; -import org.globus.cog.abstraction.impl.common.taskgraph.DependencyPair; -import org.globus.cog.abstraction.interfaces.Dependency; -import org.globus.cog.abstraction.interfaces.ExecutableObject; -import org.globus.cog.abstraction.interfaces.Status; -import org.globus.cog.abstraction.interfaces.StatusListener; -import org.globus.cog.abstraction.interfaces.Task; -import org.globus.cog.abstraction.interfaces.TaskGraph; -import org.globus.cog.gui.grapheditor.RendererFactory; -import org.globus.cog.gui.grapheditor.RootContainer; -import org.globus.cog.gui.grapheditor.canvas.GraphCanvas; -import org.globus.cog.gui.grapheditor.generic.GenericEdge; -import org.globus.cog.gui.grapheditor.generic.GenericNode; -import org.globus.cog.gui.grapheditor.generic.RootNode; -import org.globus.cog.gui.grapheditor.properties.OverlayedProperty; -import org.globus.cog.gui.grapheditor.properties.Property; -import org.globus.cog.gui.grapheditor.targets.swing.GraphFrame; -import org.globus.cog.gui.grapheditor.targets.swing.SwingCanvasRenderer; -import org.globus.cog.gui.grapheditor.targets.swing.util.CanvasAction; -import org.globus.cog.gui.grapheditor.targets.swing.util.CanvasActionEvent; -import org.globus.cog.gui.grapheditor.targets.swing.util.CanvasActionListener; -import org.globus.cog.gui.grapheditor.targets.swing.views.GraphView; -import org.globus.cog.util.graph.Graph; -import org.globus.cog.util.graph.Node; - -public class HierarchicalDAGVisualizer extends HierarchicalDAG implements - StatusListener, ActionListener, CanvasActionListener { - private Hashtable nodeMapping; - private CanvasAction run; - - public HierarchicalDAGVisualizer() { - super(); - this.nodeMapping = new Hashtable(); - } - - private RootNode init() { - RootNode rootNode = new RootNode(); - rootNode.setName("Java CoG Kit Work Manager"); - GraphCanvas graphCanvas = rootNode.getCanvas(); - if (graphCanvas == null) { - graphCanvas = rootNode.createCanvas(); - } - Graph graph = new Graph(); - GenericNode gn = new GenericNode(); - createSubGraph(gn, this.taskGraph); - graph.addNode(gn); - this.nodeMapping.put(this.taskGraph, gn); - graphCanvas.setGraph(graph); - return rootNode; - } - - private void updateRenderer(SwingCanvasRenderer renderer) { - this.run = new CanvasAction("300#Control>1#Run", null, - CanvasAction.ACTION_NORMAL); - this.run.addCanvasActionListener(this); - renderer.addToolBarItem(this.run); - - renderer.setView(new GraphView()); - } - - private void createSubGraph(GenericNode genericNode, - ExecutableObject executableObject) { - if (executableObject.getObjectType() == ExecutableObject.TASK) { - Task task = (Task) executableObject; - task.addStatusListener(this); - setTaskProperties(genericNode, task); - } else if (executableObject.getObjectType() == ExecutableObject.TASKGRAPH) { - TaskGraph taskGraph = (TaskGraph) executableObject; - taskGraph.addStatusListener(this); - setGraphProperties(genericNode, taskGraph); - - GraphCanvas subCanvas = genericNode.getCanvas(); - if (subCanvas == null) { - subCanvas = genericNode.createCanvas(); - } - Graph graph = new Graph(); - - Enumeration en = taskGraph.elements(); - while (en.hasMoreElements()) { - ExecutableObject eo = (ExecutableObject) en.nextElement(); - GenericNode gn = new GenericNode(); - createSubGraph(gn, eo); - graph.addNode(gn); - this.nodeMapping.put(eo, gn); - } - - // Draw the dependencies - Dependency dependency = taskGraph.getDependency(); - if (dependency != null) { - Enumeration en1 = dependency.elements(); - while (en1.hasMoreElements()) { - DependencyPair pair = (DependencyPair) en1.nextElement(); - ExecutableObject eo1 = pair.getFrom(); - ExecutableObject eo2 = pair.getTo(); - - GenericNode gn1 = (GenericNode) this.nodeMapping.get(eo1); - GenericNode gn2 = (GenericNode) this.nodeMapping.get(eo2); - - Node node1 = graph.findNode(gn1); - Node node2 = graph.findNode(gn2); - - graph.addEdge(node1, node2, new GenericEdge()); - } - } - subCanvas.setGraph(graph); - } - } - - private void setTaskProperties(GenericNode genericNode, Task task) { - Property property; - - property = new OverlayedProperty(genericNode, "Identity"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Type"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "StdError"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "StdOutput"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Provider"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "ServiceContact"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Specification"); - genericNode.addProperty(property); - - // set all the above defined properties: - genericNode.getProperty("name").setValue(task.getName()); - genericNode.getProperty("Identity").setValue( - task.getIdentity().toString()); - genericNode.getProperty("Type").setValue( - Integer.toString(task.getType())); - genericNode.getProperty("StdOutput").setValue(task.getStdOutput()); - genericNode.getProperty("StdError").setValue(task.getStdError()); - genericNode.getProperty("Provider").setValue(task.getProvider()); - genericNode.getProperty("status").setValue( - new Integer(GenericNode.STATUS_STOPPED)); - genericNode.getProperty("Specification").setValue( - task.getSpecification().getSpecification()); - } - - private void setGraphProperties(GenericNode genericNode, TaskGraph taskGraph) { - Property property; - - property = new OverlayedProperty(genericNode, "Identity"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Size"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Unsubmitted"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Submitted"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Active"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Suspended"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Resumed"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Canceled"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Failed"); - genericNode.addProperty(property); - - property = new OverlayedProperty(genericNode, "Completed"); - genericNode.addProperty(property); - - // set all the above defined properties: - genericNode.getProperty("name").setValue(taskGraph.getName()); - genericNode.getProperty("Identity").setValue( - taskGraph.getIdentity().toString()); - genericNode.getProperty("status").setValue( - new Integer(GenericNode.STATUS_STOPPED)); - genericNode.getProperty("Size").setValue( - Integer.toString(taskGraph.getSize())); - genericNode.getProperty("Unsubmitted").setValue( - Integer.toString(taskGraph.getUnsubmittedCount())); - genericNode.getProperty("Submitted").setValue( - Integer.toString(taskGraph.getSubmittedCount())); - genericNode.getProperty("Active").setValue( - Integer.toString(taskGraph.getActiveCount())); - genericNode.getProperty("Suspended").setValue( - Integer.toString(taskGraph.getSuspendedCount())); - genericNode.getProperty("Resumed").setValue( - Integer.toString(taskGraph.getResumedCount())); - genericNode.getProperty("Canceled").setValue( - Integer.toString(taskGraph.getCanceledCount())); - genericNode.getProperty("Failed").setValue( - Integer.toString(taskGraph.getFailedCount())); - genericNode.getProperty("Completed").setValue( - Integer.toString(taskGraph.getCompletedCount())); - } - - public void updateProperties(StatusEvent event) { - TaskGraph taskGraph = (TaskGraph) event.getSource(); - GenericNode genericNode = (GenericNode) this.nodeMapping.get(event - .getSource()); - - genericNode.getProperty("Unsubmitted").setValue( - Integer.toString(taskGraph.getUnsubmittedCount())); - genericNode.getProperty("Submitted").setValue( - Integer.toString(taskGraph.getSubmittedCount())); - genericNode.getProperty("Active").setValue( - Integer.toString(taskGraph.getActiveCount())); - genericNode.getProperty("Suspended").setValue( - Integer.toString(taskGraph.getSuspendedCount())); - genericNode.getProperty("Resumed").setValue( - Integer.toString(taskGraph.getResumedCount())); - genericNode.getProperty("Canceled").setValue( - Integer.toString(taskGraph.getCanceledCount())); - genericNode.getProperty("Failed").setValue( - Integer.toString(taskGraph.getFailedCount())); - genericNode.getProperty("Completed").setValue( - Integer.toString(taskGraph.getCompletedCount())); - } - - public void statusChanged(StatusEvent event) { - ExecutableObject executableObject = event.getSource(); - GenericNode gn = (GenericNode) this.nodeMapping.get(executableObject); - int status = event.getStatus().getStatusCode(); - logger.debug("ID: " + executableObject.getIdentity().toString()); - logger.debug("Status: " + executableObject.getStatus().getStatusCode()); - switch (status) { - case Status.UNSUBMITTED: - gn.getProperty("status").setValue( - new Integer(GenericNode.STATUS_STOPPED)); - break; - case Status.SUBMITTED: - gn.getProperty("status").setValue( - new Integer(GenericNode.STATUS_RUNNING)); - break; - case Status.ACTIVE: - gn.getProperty("status").setValue( - new Integer(GenericNode.STATUS_RUNNING)); - break; - case Status.FAILED: - gn.getProperty("status").setValue( - new Integer(GenericNode.STATUS_FAILED)); - break; - case Status.COMPLETED: - gn.getProperty("status").setValue( - new Integer(GenericNode.STATUS_COMPLETED)); - break; - case Status.CANCELED: - gn.getProperty("status").setValue( - new Integer(GenericNode.STATUS_FAILED)); - break; - case Status.SUSPENDED: - gn.getProperty("status").setValue( - new Integer(GenericNode.STATUS_STOPPED)); - break; - case Status.RESUMED: - gn.getProperty("status").setValue( - new Integer(GenericNode.STATUS_RUNNING)); - break; - default: - break; - } - if (executableObject.getObjectType() == ExecutableObject.TASKGRAPH) { - updateProperties(event); - } - } - - public void actionPerformed(ActionEvent event) { - if (event.getActionCommand().equals("Run")) { - if (!this.active) { - this.active = true; - submitDAG(); - } - } - } - - public void canvasActionPerformed(CanvasActionEvent e) { - if (e.getSource() == this.run) { - if (!this.active) { - this.active = true; - submitDAG(); - } - } - } - - public static void main(String[] args) { - try { - HierarchicalDAGVisualizer dagViz = new HierarchicalDAGVisualizer(); - dagViz.createDAG(); - RendererFactory.addRootContainer("swing", GraphFrame.class); - RootContainer rootContainer = new GraphFrame(); - RootNode rootNode = dagViz.init(); - rootContainer.setRootNode(rootNode); - dagViz.updateRenderer((SwingCanvasRenderer) rootContainer - .getCanvasRenderer()); - Thread thread = new Thread(rootContainer); - thread.start(); - } catch (Exception e) { - logger.error("Exception caught: ", e); - } - } -} \ No newline at end of file Index: modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/HierarchicalDAG.java =================================================================== --- modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/HierarchicalDAG.java (revision 4012) +++ modules/examples/src/org/globus/cog/abstraction/examples/taskgraph/HierarchicalDAG.java (working copy) @@ -1,182 +0,0 @@ -// ---------------------------------------------------------------------- -// This code is developed as part of the Java CoG Kit project -// The terms of the license can be found at http://www.cogkit.org/license -// This message may not be removed or altered. -// ---------------------------------------------------------------------- - -package org.globus.cog.abstraction.examples.taskgraph; - -import org.apache.log4j.Logger; -import org.globus.cog.abstraction.impl.common.StatusEvent; -import org.globus.cog.abstraction.impl.common.task.IllegalSpecException; -import org.globus.cog.abstraction.impl.common.task.InvalidSecurityContextException; -import org.globus.cog.abstraction.impl.common.task.InvalidServiceContactException; -import org.globus.cog.abstraction.impl.common.task.TaskSubmissionException; -import org.globus.cog.abstraction.impl.common.taskgraph.TaskGraphHandlerImpl; -import org.globus.cog.abstraction.impl.common.taskgraph.TaskGraphImpl; -import org.globus.cog.abstraction.interfaces.ExecutableObject; -import org.globus.cog.abstraction.interfaces.Status; -import org.globus.cog.abstraction.interfaces.StatusListener; -import org.globus.cog.abstraction.interfaces.Task; -import org.globus.cog.abstraction.interfaces.TaskGraph; -import org.globus.cog.abstraction.interfaces.TaskGraphHandler; -import org.globus.cog.abstraction.tools.transfer.FileTransfer; - -/* - * This class serves as an example to demonstrate the execution of hierarchical - * taskgraphs (DAGs). It simply shows the semantics of how a user can create a - * hierarchy of task graph and submit additional tasks to an already executing - * task graph (a.k.a adding tasks to a "live taskgraph"). The parameters to the - * task specification MUST be changed to suite your requirements and - * environment. - */ -public class HierarchicalDAG implements StatusListener { - static Logger logger = Logger.getLogger(HierarchicalDAG.class.getName()); - protected org.globus.cog.abstraction.interfaces.TaskGraph taskGraph; - protected boolean active = false; - private boolean submitted = false; - private Task task6 = null; - private Task task9 = null; - private TaskGraph tg5 = null; - - public void createDAG() throws Exception { - TaskGraph tg1 = new TaskGraphImpl(); - try { - tg1.setName("TG1"); - Task task1 = prepareTask("Task1", - "gridftp://hot.mcs.anl.gov:2811//home/cog/gridfile1", - "gridftp://cold.mcs.anl.gov:2811//home/cog/gridfile1"); - Task task2 = prepareTask("Task2", - "gridftp://cold.mcs.anl.gov:2811//home/cog/gridfile2", - "gridftp://hot.mcs.anl.gov:2811//home/cog/gridfile2"); - tg1.add(task1); - tg1.add(task2); - tg1.addDependency(task1, task2); - - TaskGraph tg2 = new TaskGraphImpl(); - tg2.setName("TG2"); - Task task3 = prepareTask("Task3", - "gridftp://hot.mcs.anl.gov:2811//home/cog/gridfile3", - "gridftp://cold.mcs.anl.gov:2811//home/cog/gridfile3"); - Task task4 = prepareTask("Task4", - "gridftp://cold.mcs.anl.gov:2811//home/cog/gridfile4", - "gridftp://hot.mcs.anl.gov:2811//home/cog/gridfile4"); - tg2.add(task3); - tg2.add(task4); - tg2.addDependency(task3, task4); - - TaskGraph tg3 = new TaskGraphImpl(); - tg3.setName("TG3"); - tg3.add(tg1); - tg3.add(tg2); - tg3.addDependency(tg1, tg2); - - TaskGraph tg4 = new TaskGraphImpl(); - tg4.setName("TG4"); - Task task5 = prepareTask("Task5", - "gridftp://new.mcs.anl.gov:2811//home/cog/gridfile6", - "gridftp://old.mcs.anl.gov:2811//home/cog/gridfile7"); - tg4.add(tg3); - tg4.add(task5); - tg4.addDependency(tg3, task5); - - tg5 = new TaskGraphImpl(); - - Task task7 = prepareTask("Task7", - "gridftp://old.mcs.anl.gov:2811//home/cog/gridfile8", - "gridftp://new.mcs.anl.gov:2811//home/cog/gridfile9"); - Task task8 = prepareTask("Task8", - "gridftp://here.mcs.anl.gov:2811//home/cog/gridfile10", - "gridftp://there.mcs.anl.gov:2811//home/cog/gridfile10"); - task9 = prepareTask("Task9", - "gridftp://here.mcs.anl.gov:2811//home/cog/gridfile11", - "gridftp://there.mcs.anl.gov:2811//home/cog/gridfile11"); - - tg5.add(task7); - tg5.add(task8); - tg5.add(task9); - tg5.add(tg4); - - tg5.addDependency(task7, task8); - tg5.addDependency(task8, task9); - tg5.addDependency(task8, tg4); - tg5.addDependency(task9, tg4); - - this.taskGraph = tg5; - this.taskGraph.setName("Main Graph"); - this.taskGraph.addStatusListener(this); - } catch (Exception e) { - logger.error("Unable to create DAG", e); - } - } - - public void submitDAG() { - TaskGraphHandler handler = new TaskGraphHandlerImpl(); - try { - handler.submit(this.taskGraph); - logger.debug("TaskGraph submitted"); - } catch (InvalidSecurityContextException ise) { - logger.error("Security Exception"); - ise.printStackTrace(); - System.exit(1); - } catch (TaskSubmissionException tse) { - logger.error("TaskSubmission Exception"); - tse.printStackTrace(); - System.exit(1); - } catch (IllegalSpecException ispe) { - logger.error("Specification Exception"); - ispe.printStackTrace(); - System.exit(1); - } catch (InvalidServiceContactException isce) { - logger.error("Service Contact Exception"); - isce.printStackTrace(); - System.exit(1); - } - } - - private Task prepareTask(String name, String source, String destination) - throws Exception { - FileTransfer fileTransfer = new FileTransfer(name, source, destination); - fileTransfer.prepareTask(); - Task task = fileTransfer.getFileTransferTask(); - task.removeStatusListener(fileTransfer); - return task; - } - - public void statusChanged(StatusEvent event) { - ExecutableObject eo = event.getSource(); - logger.debug(eo.getName()); - Status status = event.getStatus(); - logger.debug("Status changed to: " + status.getStatusString()); - - // demonstrating "live" task graphs - if (status.getStatusCode() == Status.SUBMITTED && !submitted) { - submitted = true; - try { - task6 = prepareTask("Task6", - "gridftp://this.mcs.anl.gov:2811//home/cog/gridfile12", - "gridftp://that.mcs.anl.gov:2811//home/cog/gridfile12"); - tg5.addDependency(task6, task9); - tg5.add(task6); - } catch (Exception e) { - e.printStackTrace(); - } - } - - if (status.getStatusCode() == Status.COMPLETED - || status.getStatusCode() == Status.FAILED) { - logger.info("Task Graph Done"); - // System.exit(1); - } - } - - public static void main(String arg[]) { - try { - HierarchicalDAG dag = new HierarchicalDAG(); - dag.createDAG(); - dag.submitDAG(); - } catch (Exception e) { - logger.error("Exception caught: ", e); - } - } -} \ No newline at end of file Index: modules/examples/etc/MANIFEST.MF.tail =================================================================== --- modules/examples/etc/MANIFEST.MF.tail (revision 4012) +++ modules/examples/etc/MANIFEST.MF.tail (working copy) @@ -1 +0,0 @@ - Index: modules/examples/etc/MANIFEST.MF.head =================================================================== --- modules/examples/etc/MANIFEST.MF.head (revision 4012) +++ modules/examples/etc/MANIFEST.MF.head (working copy) @@ -1 +0,0 @@ -Manifest-Version: 1.0 Index: modules/examples/build.xml =================================================================== --- modules/examples/build.xml (revision 4012) +++ modules/examples/build.xml (working copy) @@ -1,152 +0,0 @@ - - - - - - - - - - - - - - - - Available targets: - help: - prints out this help message - - dist: - creates a distribution directory of the - ${project} ${long.name} - - jar: - creates a jar file for the ${project} ${long.name} - named ${jar.filename} - - javadoc: - creates the documentation - - clean: - removes the compiled classes - - distclean: - deletes the distribution directory - - all: - dist and javadoc - - deploy.webstart: - deploys the module as a webstart application - - dist.joint: - builds everything into one jar file. Should only - be used globally (from all) - - fixeol: - change newlines to the unix standard - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From hategan at ci.uchicago.edu Fri Jul 4 02:36:12 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:36:12 -0500 (CDT) Subject: [Swift-commit] r7968 - trunk/resources Message-ID: <20140704073612.34DC39D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 02:36:11 -0500 (Fri, 04 Jul 2014) New Revision: 7968 Modified: trunk/resources/swift-sites-2.0.xsd Log: updated sites spec a bit Modified: trunk/resources/swift-sites-2.0.xsd =================================================================== --- trunk/resources/swift-sites-2.0.xsd 2014-07-04 07:01:55 UTC (rev 7967) +++ trunk/resources/swift-sites-2.0.xsd 2014-07-04 07:36:11 UTC (rev 7968) @@ -39,7 +39,7 @@ - + From hategan at ci.uchicago.edu Fri Jul 4 02:36:41 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:36:41 -0500 (CDT) Subject: [Swift-commit] r7969 - trunk/src/org/griphyn/vdl/karajan/lib Message-ID: <20140704073641.263099D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 02:36:40 -0500 (Fri, 04 Jul 2014) New Revision: 7969 Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java Log: global apps consistent with site app format Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-07-04 07:36:11 UTC (rev 7968) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-07-04 07:36:40 UTC (rev 7969) @@ -99,9 +99,16 @@ throw new ExecutionException(this, "Invalid site entry '" + poolName(n) + "': ", e); } } - else if (ctype.equals("application")) { - cs.addApplication(application(n)); + else if (ctype.equals("apps")) { + SwiftContact dummy = new SwiftContact(); + apps(dummy, n); + for (Application app : dummy.getApplications()) { + cs.addApplication(app); + } } + else { + throw new IllegalArgumentException("Invalid node: " + ctype); + } } } return cs; From hategan at ci.uchicago.edu Fri Jul 4 02:49:46 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 02:49:46 -0500 (CDT) Subject: [Swift-commit] r7970 - trunk Message-ID: <20140704074946.E3F869D81D@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 02:49:45 -0500 (Fri, 04 Jul 2014) New Revision: 7970 Modified: trunk/launchers.xml Log: added swift-log-info launcher Modified: trunk/launchers.xml =================================================================== --- trunk/launchers.xml 2014-07-04 07:36:40 UTC (rev 7969) +++ trunk/launchers.xml 2014-07-04 07:49:45 UTC (rev 7970) @@ -8,6 +8,10 @@ + + + + From swift at ci.uchicago.edu Fri Jul 4 03:00:06 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Fri, 4 Jul 2014 03:00:06 -0500 (CDT) Subject: [Swift-commit] cog r4014 Message-ID: <20140704080006.AE5A88D000A5@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r4014 | hategan | 2014-07-04 02:55:52 -0500 (Fri, 04 Jul 2014) | 1 line changed probe interval to 60 seconds ------------------------------------------------------------------------ Index: modules/provider-coaster/resources/worker.pl =================================================================== --- modules/provider-coaster/resources/worker.pl (revision 4013) +++ modules/provider-coaster/resources/worker.pl (working copy) @@ -166,7 +166,7 @@ # Contains tuples (EVENT, PID, TIMESTAMP) (flattened) my @PROFILE_EVENTS = (); -my $PROBE_INTERVAL = 1; +my $PROBE_INTERVAL = 60; my $LAST_PROBE_TIME = time() - $PROBE_INTERVAL + 1; my $ID = "-"; From hategan at ci.uchicago.edu Fri Jul 4 13:09:44 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 13:09:44 -0500 (CDT) Subject: [Swift-commit] r7971 - trunk/src/org/griphyn/vdl/karajan Message-ID: <20140704180944.23A9B9DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 13:09:43 -0500 (Fri, 04 Jul 2014) New Revision: 7971 Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java Log: removed -monitor and -typecheck and cleaned up command line arg info a bit Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-04 07:49:45 UTC (rev 7970) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-04 18:09:43 UTC (rev 7971) @@ -78,7 +78,6 @@ public static final String ARG_VERSION = "version"; public static final String ARG_RESUME = "resume"; public static final String ARG_INSTANCE_CONFIG = "config"; - public static final String ARG_TYPECHECK = "typecheck"; public static final String ARG_DRYRUN = "dryrun"; public static final String ARG_VERBOSE = "verbose"; public static final String ARG_DEBUG = "debug"; @@ -92,11 +91,6 @@ public static final String ARG_PAUSE_ON_START = "pause.on.start"; public static final String ARG_EXECUTE = "e"; - public static final String CONST_VDL_OPERATION = "vdl:operation"; - public static final String VDL_OPERATION_RUN = "run"; - public static final String VDL_OPERATION_TYPECHECK = "typecheck"; - public static final String VDL_OPERATION_DRYRUN = "dryrun"; - public static String buildVersion; public static void main(String[] argv) { @@ -565,69 +559,51 @@ ArgumentParser ap = new ArgumentParser(); ap.setArguments(true); ap.setExecutableName("swift"); - ap - .addOption(ArgumentParser.DEFAULT, - "A file (.swift or .kml) to execute", "file", - ArgumentParser.OPTIONAL); + ap.addOption(ArgumentParser.DEFAULT, + "A file (.swift or .kml) to execute", "file", + ArgumentParser.OPTIONAL); ap.addFlag(ARG_HELP, "Display usage information"); - ap.addFlag(ARG_VERSION, "Version:"); + ap.addFlag(ARG_VERSION, "Version:"); ap.addAlias(ARG_HELP, "h"); ap.addFlag(ARG_RECOMPILE, "Forces Swift to re-compile the invoked Swift script. " + "While Swift is meant to detect when recompilation is necessary, " + "in some special cases it fails to do so. This flag helps with those special cases."); - ap.addFlag(ARG_TYPECHECK, - "Does a typecheck instead of executing the SwiftScript program"); - ap - .addFlag( - ARG_DRYRUN, - "Runs the SwiftScript program without submitting any jobs (can be used to get a graph)"); + ap.addFlag(ARG_DRYRUN, + "Runs the SwiftScript program without submitting any jobs (can be used to get a graph)"); - ap.addFlag(ARG_MONITOR, "Shows a graphical resource monitor"); - ap.addOption(ARG_RESUME, "Resumes the execution using a log file", "file", ArgumentParser.OPTIONAL); - ap - .addOption( - ARG_INSTANCE_CONFIG, - "Indicates the Swift configuration file to be used for this run." - + " Properties in this configuration file will override the default properties. " - + "If individual command line arguments are used for properties, they will override " - + "the contents of this file.", "file", - ArgumentParser.OPTIONAL); - ap - .addFlag( - ARG_VERBOSE, - "Increases the level of output that Swift produces on the console to include more detail " - + "about the execution"); + ap.addOption(ARG_INSTANCE_CONFIG, + "Indicates the Swift configuration file to be used for this run." + + " Properties in this configuration file will override the default properties. " + + "If individual command line arguments are used for properties, they will override " + + "the contents of this file.", "file", + ArgumentParser.OPTIONAL); + ap.addFlag(ARG_VERBOSE, + "Increases the level of output that Swift produces on the console to include more detail " + + "about the execution"); ap.addAlias(ARG_VERBOSE, "v"); - ap - .addFlag( - ARG_DEBUG, - "Increases the level of output that Swift produces on the console to include lots of " - + "detail about the execution"); + ap.addFlag(ARG_DEBUG, + "Increases the level of output that Swift produces on the console to include lots of " + + "detail about the execution"); ap.addAlias(ARG_DEBUG, "d"); - ap - .addOption( - ARG_LOGFILE, - "Specifies a file where log messages should go to. By default Swift " - + "uses the name of the SwiftScript program being run and additional information to make the name unique.", - "file", ArgumentParser.OPTIONAL); - ap - .addOption( - ARG_CDMFILE, + ap.addOption(ARG_LOGFILE, + "Specifies a file where log messages should go to. By default Swift " + + "uses the name of the SwiftScript program being run and additional information to make the name unique.", + "file", ArgumentParser.OPTIONAL); + ap.addOption(ARG_CDMFILE, "Specifies a CDM policy file.", "file", ArgumentParser.OPTIONAL); - ap - .addOption( - ARG_RUNID, - "Specifies the run identifier. This must be unique for every invocation of a workflow and is used in several places to keep files from different executions cleanly separated. By default, a datestamp and random number are used to generate a run identifier.", - "string", ArgumentParser.OPTIONAL); - ap.addOption( - ARG_UI, + ap.addOption(ARG_RUNID, + "Specifies the run identifier. This must be unique for every invocation of a script and " + + "is used in several places to keep files from different runs cleanly separated. " + + "By default, a datestamp and random number are used to generate a run identifier.", + "string", ArgumentParser.OPTIONAL); + ap.addOption(ARG_UI, "Indicates how swift should display run-time information. The following are valid values:" + "\n\t'summary' (default) - causesSswift to regularly print a count of jobs for each state that a job can be in" + "\n\t'text' - regularly prints a more detailed table with Swift run-time information" + @@ -637,11 +613,11 @@ "\n\t'http' - enables an http server allowing access to swift run-time information using a web browser", "", ArgumentParser.OPTIONAL); ap.addFlag(ARG_REDUCED_LOGGING, "Makes logging more terse by disabling provenance " + - "information and low-level task messages"); + "information and low-level task messages"); ap.addFlag(ARG_MINIMAL_LOGGING, "Makes logging much more terse: " + - "reports warnings only"); + "reports warnings only"); ap.addFlag(ARG_PAUSE_ON_START, "Pauses execution on start. Useful for " + - "attaching a debugger or profiler to the swift process"); + "attaching a debugger or profiler to the swift process"); ap.addOption(ARG_EXECUTE, "Runs the swift script code contained in ", "string", ArgumentParser.OPTIONAL); From hategan at ci.uchicago.edu Fri Jul 4 13:28:22 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 13:28:22 -0500 (CDT) Subject: [Swift-commit] r7972 - trunk/libexec Message-ID: <20140704182822.0B13E9DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 13:28:21 -0500 (Fri, 04 Jul 2014) New Revision: 7972 Modified: trunk/libexec/_swiftwrap trunk/libexec/swift-int.k trunk/libexec/swift.k Log: removed kickstart Modified: trunk/libexec/_swiftwrap =================================================================== --- trunk/libexec/_swiftwrap 2014-07-04 18:09:43 UTC (rev 7971) +++ trunk/libexec/_swiftwrap 2014-07-04 18:28:21 UTC (rev 7972) @@ -480,73 +480,49 @@ if [ ! -x "$EXEC" ]; then fail 254 "The executable $EXEC does not have the executable bit set" fi -if [ "$KICKSTART" == "" ]; then - TIMEARGS=(-o swiftapp.resources -f APP_RESOURCES=real_secs:%e,kernel_secs:%S,user_secs:%U,percent_cpu:%P,max_rss:%M,avg_rss:%t,avg_tot_vm:%K,avg_priv_data:%D,avg_priv_stack:%p,avg_shared_text:%X,page_size:%Z,major_pgfaults:%F,minor_pgfaults:%R,swaps:%W,invol_context_switches:%c,vol_waits:%w,fs_reads:%I,fs_writes:%O,sock_recv:%r,sock_send:%s,signals:%k,exit_status:%x) +TIMEARGS=(-o swiftapp.resources -f APP_RESOURCES=real_secs:%e,kernel_secs:%S,user_secs:%U,percent_cpu:%P,max_rss:%M,avg_rss:%t,avg_tot_vm:%K,avg_priv_data:%D,avg_priv_stack:%p,avg_shared_text:%X,page_size:%Z,major_pgfaults:%F,minor_pgfaults:%R,swaps:%W,invol_context_switches:%c,vol_waits:%w,fs_reads:%I,fs_writes:%O,sock_recv:%r,sock_send:%s,signals:%k,exit_status:%x) - if [[ "$OSTYPE" == *darwin* ]]; then - TIMECMD= - TIMEARGS= - elif [ -x /usr/bin/time ]; then - TIMECMD="/usr/bin/time" - elif [ -x $HOME/swift.time ]; then - TIMECMD="$HOME/swift.time" - else - TIMECMD="" - TIMEARGS="" - fi - if [ "$STDIN" == "" ]; then - if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then - genScripts - fi - - if [ -n "$TIMECMD" ] && [ -n "$TIMEARGS" ]; then - "$TIMECMD" "${TIMEARGS[@]}" "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" - else - "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" - fi - else - if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then - genScripts - fi - - if [ -n "$TIMECMD" ] && [ -n "$TIMEARGS" ]; then - "$TIMECMD" "${TIMEARGS[@]}" "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" <"$STDIN" - else - "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" <"$STDIN" - fi +if [[ "$OSTYPE" == *darwin* ]]; then + TIMECMD= + TIMEARGS= +elif [ -x /usr/bin/time ]; then + TIMECMD="/usr/bin/time" +elif [ -x $HOME/swift.time ]; then + TIMECMD="$HOME/swift.time" +else + TIMECMD="" +TIMEARGS="" +fi +if [ "$STDIN" == "" ]; then + if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then + genScripts fi - - TIME_EC=$? - if [ "_$TIMECMD" != _ ]; then - log "$(cat swiftapp.resources)" - fi - sh -c "exit $TIME_EC" - checkError $? "Application $EXEC failed with an exit code of $?" + + if [ -n "$TIMECMD" ] && [ -n "$TIMEARGS" ]; then + "$TIMECMD" "${TIMEARGS[@]}" "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" + else + "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" + fi else - if [ ! -f "$KICKSTART" ]; then - log "Kickstart executable ($KICKSTART) not found" - fail 254 "The Kickstart executable ($KICKSTART) was not found" - elif [ ! -x "$KICKSTART" ]; then - log "Kickstart executable ($KICKSTART) is not executable" - fail 254 "The Kickstart executable ($KICKSTART) does not have the executable bit set" - else - mkdir -p $WFDIR/kickstart/$JOBDIR - log "Using Kickstart ($KICKSTART)" - if [ "$STDIN" == "" ]; then - "$KICKSTART" -H -o "$STDOUT" -e "$STDERR" "$EXEC" "$@" 1>kickstart.xml 2>>"$STDERR" - else - "$KICKSTART" -H -o "$STDOUT" -i "$STDIN" -e "$STDERR" "$EXEC" "$@" 1>kickstart.xml 2>>"$STDERR" - fi - export APPEXIT=$? - mv -f kickstart.xml "$WFDIR/kickstart/$JOBDIR/$ID-kickstart.xml" 2>&1 >& "$INFO" - checkError 254 "Failed to copy Kickstart record to shared directory" - if [ "$APPEXIT" != "0" ]; then - fail $APPEXIT "Application $EXEC failed with an exit code of $APPEXIT" - fi + if [ "$SWIFT_GEN_SCRIPTS" != "" ]; then + genScripts fi + + if [ -n "$TIMECMD" ] && [ -n "$TIMEARGS" ]; then + "$TIMECMD" "${TIMEARGS[@]}" "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" <"$STDIN" + else + "$EXEC" "${CMDARGS[@]}" 1>>"$STDOUT" 2>>"$STDERR" <"$STDIN" + fi fi +TIME_EC=$? +if [ "_$TIMECMD" != _ ]; then + log "$(cat swiftapp.resources)" +fi +sh -c "exit $TIME_EC" +checkError $? "Application $EXEC failed with an exit code of $?" + if [[ $MPI_RANK == "" || $MPI_RANK == 0 ]]; then if [ "$COLLECT" != "" ]; then logstate "COLLECT" Modified: trunk/libexec/swift-int.k =================================================================== --- trunk/libexec/swift-int.k 2014-07-04 18:09:43 UTC (rev 7971) +++ trunk/libexec/swift-int.k 2014-07-04 18:28:21 UTC (rev 7972) @@ -85,7 +85,6 @@ dir:make(SHARED_DIR, host = rhost) transfer(siteProfile(rhost, "wrapperScript"), srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) transfer("_swiftseq", srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) - dir:make(dircat(RUN_DIR, "kickstart"), host=rhost) statusMode := configProperty("status.mode", host=rhost) if (statusMode == "files") { Modified: trunk/libexec/swift.k =================================================================== --- trunk/libexec/swift.k 2014-07-04 18:09:43 UTC (rev 7971) +++ trunk/libexec/swift.k 2014-07-04 18:28:21 UTC (rev 7972) @@ -87,59 +87,13 @@ * all the data in memory until the workflow is done */ export(mainp, - CBFFunction(channel(graph), channel(cleanup)) { + CBFFunction(channel(cleanup)) { parallel( - if(configProperty("pgraph") != "false") { - generateProvenanceGraph(graph) - } to(cleanup, unique(for(c, cleanup, c))) ) } - ) + ) - graphStuff := function(tr, stagein, stageout, err, args = null) { - if (configProperty("pgraph") != "false") { - errprops := if(err, ",color=lightsalmon", ",color=lightsteelblue1") - tp := currentThread() - to (graph) { - concat(str:quote(tp), " [label=", str:quote(tr), "{errprops}]") - } - for (si, stagein) { - si := basename(si) - to(graph) { - concat(str:quote(si), " [shape=parallelogram]") - concat(str:quote(si), " -> ", str:quote(tp)) - } - } - for (pv, stageout) { - (path, var) := each(pv) - file := fileName(getField(var, path=path)) - file := basename(file) - label := niceName(var, path = path) - to(graph) { - concat(str:quote(file), " [shape=parallelogram,label=", str:quote(label), "]") - concat(str:quote(tp), " -> ", str:quote(file)) - } - } - } - } - - generateProvenanceGraph := function(gdata) { - pgraph := configProperty("pgraph") - gname := if(pgraph == "true", "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.dot", pgraph) - file:write(gname) { - "digraph SwiftProvenance \{\n", - " graph [", configProperty("pgraph.graph.options"), "];\n", - " node [", configProperty("pgraph.node.options"), "];\n", - - for(i, gdata) { - " ", i, "\n" - } - "}\n" - } - log(LOG:INFO, "Provenance graph saved in ", gname) - } - export(execute, function( tr, arguments = null, @@ -223,7 +177,6 @@ log(LOG:INFO, exception) echo(exception) mark(stageout, true, mapping=false) - graphStuff(tr, stagein, stageout, true, maybe(args=arguments)) } } } @@ -245,7 +198,6 @@ log(LOG:INFO, exception) } mark(stageout, true, mapping=merr) - graphStuff(tr, stagein, stageout, true, maybe(args=arguments)) } } ) From hategan at ci.uchicago.edu Fri Jul 4 13:29:09 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Fri, 4 Jul 2014 13:29:09 -0500 (CDT) Subject: [Swift-commit] r7973 - in trunk/src: . org/griphyn/vdl/util Message-ID: <20140704182909.B1B659DA66@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-04 13:29:09 -0500 (Fri, 04 Jul 2014) New Revision: 7973 Modified: trunk/src/ trunk/src/org/griphyn/vdl/util/VDL2Config.java Log: removed kickstart and graph options; we can get graph information from the log Property changes on: trunk/src ___________________________________________________________________ Modified: svn:mergeinfo - /branches/release-0.94/src:6283-6619,6999-7001,7125 /trunk/src:6214,6255,6275-6618 + /branches/release-0.94/src:6283-6619,6999-7001,7125 /branches/release-0.95/src:7921 /trunk/src:6214,6255,6275-6618 Modified: trunk/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2014-07-04 18:28:21 UTC (rev 7972) +++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2014-07-04 18:29:09 UTC (rev 7973) @@ -97,14 +97,9 @@ put(VDL2ConfigProperties.TC_FILE, "${swift.home}/etc/tc.data", ConfigPropertyType.FILE); put(VDL2ConfigProperties.LAZY_ERRORS, "false", ConfigPropertyType.BOOLEAN); put(VDL2ConfigProperties.CACHING_ALGORITHM, "LRU", ConfigPropertyType.STRING); - put(VDL2ConfigProperties.PGRAPH, "false", ConfigPropertyType.BOOLEAN); - put(VDL2ConfigProperties.PGRAPH_GRAPH_OPTIONS, "splines=\"compound\", rankdir=\"TB\"", ConfigPropertyType.STRING); - put(VDL2ConfigProperties.PGRAPH_NODE_OPTIONS, "color=\"seagreen\", style=\"filled\"", ConfigPropertyType.STRING); put(VDL2ConfigProperties.CLUSTERING_ENABLED, "false", ConfigPropertyType.BOOLEAN); put(VDL2ConfigProperties.CLUSTERING_QUEUE_DELAY, "4", ConfigPropertyType.INT); put(VDL2ConfigProperties.CLUSTERING_MIN_TIME, "60", ConfigPropertyType.INT); - put(VDL2ConfigProperties.KICKSTART_ENABLED, "maybe", ConfigPropertyType.choices("true", "false", "maybe")); - put(VDL2ConfigProperties.KICKSTART_ALWAYS_TRANSFER, "false", ConfigPropertyType.BOOLEAN); put("throttle.submit", "4", ConfigPropertyType.INT); put("throttle.host.submit", "2", ConfigPropertyType.INT); put("throttle.transfers", "4", ConfigPropertyType.INT); From swift at ci.uchicago.edu Wed Jul 9 13:05:02 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 9 Jul 2014 13:05:02 -0500 (CDT) Subject: [Swift-commit] cog r4016 Message-ID: <20140709180502.9F6AA8D000A4@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r4016 | timgarmstrong | 2014-07-09 13:01:28 -0500 (Wed, 09 Jul 2014) | 1 line Update to work with new configId scheme ------------------------------------------------------------------------ Index: modules/provider-coaster-c-client/src/CoasterClient.cpp =================================================================== --- modules/provider-coaster-c-client/src/CoasterClient.cpp (revision 4015) +++ modules/provider-coaster-c-client/src/CoasterClient.cpp (working copy) @@ -129,14 +129,18 @@ started = false; } -std::string CoasterClient::setOptions(Settings& s) { +string CoasterClient::setOptions(Settings& s) { LogInfo << "Setting options: " << s << endl; ServiceConfigurationCommand scc(s); scc.execute(channel); - return scc.getConfigId(); + string *id = scc.getConfigId(); + if (id == NULL) { + throw CoasterError("Did not get expected response for config"); + } + return *id; } -void CoasterClient::submit(Job& job, std::string* configId) { +void CoasterClient::submit(Job& job, const std::string* configId) { { Lock::Scoped l(lock); jobs[job.getIdentity()] = &job; } Index: modules/provider-coaster-c-client/src/CoasterClientTest.cpp =================================================================== --- modules/provider-coaster-c-client/src/CoasterClientTest.cpp (revision 4015) +++ modules/provider-coaster-c-client/src/CoasterClientTest.cpp (working copy) @@ -28,15 +28,15 @@ s.set(Settings::Key::MAX_NODES, "1"); s.set(Settings::Key::JOBS_PER_NODE, "2"); - std::string* configId = client.setOptions(s); + std::string configId = client.setOptions(s); Job j1("/bin/date"); Job j2("/bin/echo"); j2.addArgument("testing"); j2.addArgument("1, 2, 3"); - client.submit(j1, configId); - client.submit(j2, configId); + client.submit(j1, &configId); + client.submit(j2, &configId); client.waitForJob(j1); client.waitForJob(j2); Index: modules/provider-coaster-c-client/src/RunCoasterJob.cpp =================================================================== --- modules/provider-coaster-c-client/src/RunCoasterJob.cpp (revision 4015) +++ modules/provider-coaster-c-client/src/RunCoasterJob.cpp (working copy) @@ -226,7 +226,7 @@ s.set(*skey, pair.value); } - std::string* configId = client.setOptions(s); + std::string configId = client.setOptions(s); Job j(executable); @@ -258,7 +258,7 @@ j.setStderrLocation(*str); } - client.submit(j, configId); + client.submit(j, &configId); client.waitForJob(j); Index: modules/provider-coaster-c-client/src/coasters.cpp =================================================================== --- modules/provider-coaster-c-client/src/coasters.cpp (revision 4015) +++ modules/provider-coaster-c-client/src/coasters.cpp (working copy) @@ -299,14 +299,16 @@ */ coaster_rc coaster_apply_settings(coaster_client *client, - coaster_settings *settings) + coaster_settings *settings, coaster_config_id **config) COASTER_THROWS_NOTHING { - if (settings == NULL || client == NULL) { + if (settings == NULL || client == NULL || config == NULL) { return coaster_return_error(COASTER_ERROR_INVALID, "invalid arg"); } try { - client->client.setOptions(*settings); + string *configId = new string; + *configId = client->client.setOptions(*settings); + *config = configId; return COASTER_SUCCESS; } catch (const CoasterError& err) { return coaster_error_rc(err); @@ -316,6 +318,12 @@ } coaster_rc +coaster_config_id_free(coaster_config_id *config) COASTER_THROWS_NOTHING { + delete config; + return COASTER_SUCCESS; +} + +coaster_rc coaster_job_create(const char *executable, size_t executable_len, int argc, const char **argv, const size_t *arg_lens, const char *job_manager, size_t job_manager_len, @@ -595,10 +603,10 @@ } coaster_rc -coaster_submit(coaster_client *client, coaster_job *job) - COASTER_THROWS_NOTHING { +coaster_submit(coaster_client *client, const coaster_config_id *config, + coaster_job *job) COASTER_THROWS_NOTHING { try { - client->client.submit(*job); + client->client.submit(*job, config); return COASTER_SUCCESS; } catch (const CoasterError& err) { return coaster_error_rc(err); Index: modules/provider-coaster-c-client/src/JobSubmitCommand.h =================================================================== --- modules/provider-coaster-c-client/src/JobSubmitCommand.h (revision 4015) +++ modules/provider-coaster-c-client/src/JobSubmitCommand.h (working copy) @@ -13,7 +13,8 @@ private: Job* job; std::string ss; - std::string* configId; + bool haveConfigId; + std::string configId; public: static std::string NAME; JobSubmitCommand(Job* job, const std::string* configId); Index: modules/provider-coaster-c-client/src/CoasterClient.h =================================================================== --- modules/provider-coaster-c-client/src/CoasterClient.h (revision 4015) +++ modules/provider-coaster-c-client/src/CoasterClient.h (working copy) @@ -67,15 +67,17 @@ void stop(); // TODO: how long does this hold a reference to settings? - std::string* setOptions(Settings& settings); + std::string setOptions(Settings& settings); /* * Submit a job. The job should have been filled in with * all properties. The ownership of the job object stays * with the caller, but this client will retain a reference * to the job until done jobs are purged. + * + * configId: if non-null, uses specific server-side config */ - void submit(Job& job); + void submit(Job& job, const std::string* configId); /* * Wait for job to be done. Upon completion no actions Index: modules/provider-coaster-c-client/src/ServiceConfigurationCommand.cpp =================================================================== --- modules/provider-coaster-c-client/src/ServiceConfigurationCommand.cpp (revision 4015) +++ modules/provider-coaster-c-client/src/ServiceConfigurationCommand.cpp (working copy) @@ -10,7 +10,7 @@ ServiceConfigurationCommand::ServiceConfigurationCommand(Settings& s): Command(&NAME) { settings = &s; - configId = null; + configId = NULL; } void ServiceConfigurationCommand::send(CoasterChannel* channel, CommandCallback* cb) { @@ -35,6 +35,6 @@ configId = getInData()->at(0)->str(); } -std::string* getConfigId() { +std::string* ServiceConfigurationCommand::getConfigId() { return configId; } Index: modules/provider-coaster-c-client/src/coasters.h =================================================================== --- modules/provider-coaster-c-client/src/coasters.h (revision 4015) +++ modules/provider-coaster-c-client/src/coasters.h (working copy) @@ -29,6 +29,10 @@ #define COASTERS_H_ #ifdef __cplusplus +#include +#endif + +#ifdef __cplusplus extern "C" { #endif @@ -55,10 +59,12 @@ } typedef class Coaster::Settings coaster_settings; +typedef std::string coaster_config_id; typedef class Coaster::Job coaster_job; #else // Treat these types as opaque pointer to unimplemented struct for C typedef struct coaster_settings_opaque_ coaster_settings; +typedef struct coaster_config_id_opaque_ coaster_config_id; typedef struct coaster_job_opaque_ coaster_job; #endif @@ -177,12 +183,18 @@ * Apply settings to started coasters client. * TODO: currently it isn't safe to free settings until client is shut * down + * + * config: set to identifer for this service config, must be freed with + * coaster_free_config_id */ coaster_rc coaster_apply_settings(coaster_client *client, - coaster_settings *settings) + coaster_settings *settings, coaster_config_id **config) COASTER_THROWS_NOTHING; +coaster_rc +coaster_config_id_free(coaster_config_id *config) COASTER_THROWS_NOTHING; + /* * Create a new coasters job for later submission. * Some standard arguments can be specified now, or left as NULL to be @@ -303,8 +315,8 @@ * shuts down. */ coaster_rc -coaster_submit(coaster_client *client, coaster_job *job) - COASTER_THROWS_NOTHING; +coaster_submit(coaster_client *client, const coaster_config_id *config, + coaster_job *job) COASTER_THROWS_NOTHING; /* * Check for completion of jobs. Index: modules/provider-coaster-c-client/src/JobSubmitCommand.cpp =================================================================== --- modules/provider-coaster-c-client/src/JobSubmitCommand.cpp (revision 4015) +++ modules/provider-coaster-c-client/src/JobSubmitCommand.cpp (working copy) @@ -21,7 +21,13 @@ JobSubmitCommand::JobSubmitCommand(Job* pjob, const std::string* pconfigId): Command(&NAME) { job = pjob; - configId = pconfigId; + if (pconfigId != NULL) { + haveConfigId = true; + configId = *pconfigId; + } + else { + haveConfigId = false; + } } void JobSubmitCommand::send(CoasterChannel* channel, CommandCallback* cb) { @@ -45,7 +51,9 @@ void JobSubmitCommand::serialize() { stringstream idSS; idSS << job->getIdentity(); - add(ss, "configid", configId); + if (haveConfigId) { + add(ss, "configid", configId); + } add(ss, "identity", idSS.str()); add(ss, "executable", job->getExecutable()); add(ss, "directory", job->getDirectory()); From swift at ci.uchicago.edu Wed Jul 9 13:30:03 2014 From: swift at ci.uchicago.edu (swift at ci.uchicago.edu) Date: Wed, 9 Jul 2014 13:30:03 -0500 (CDT) Subject: [Swift-commit] cog r4017 Message-ID: <20140709183003.853C78D000A4@bridled.ci.uchicago.edu> ------------------------------------------------------------------------ r4017 | timgarmstrong | 2014-07-09 13:29:43 -0500 (Wed, 09 Jul 2014) | 1 line Modify argument types and documents to reflect that configId is required ------------------------------------------------------------------------ Index: modules/provider-coaster-c-client/src/CoasterClient.cpp =================================================================== --- modules/provider-coaster-c-client/src/CoasterClient.cpp (revision 4016) +++ modules/provider-coaster-c-client/src/CoasterClient.cpp (working copy) @@ -140,7 +140,7 @@ return *id; } -void CoasterClient::submit(Job& job, const std::string* configId) { +void CoasterClient::submit(Job& job, const std::string& configId) { { Lock::Scoped l(lock); jobs[job.getIdentity()] = &job; } Index: modules/provider-coaster-c-client/src/CoasterClientTest.cpp =================================================================== --- modules/provider-coaster-c-client/src/CoasterClientTest.cpp (revision 4016) +++ modules/provider-coaster-c-client/src/CoasterClientTest.cpp (working copy) @@ -35,8 +35,8 @@ j2.addArgument("testing"); j2.addArgument("1, 2, 3"); - client.submit(j1, &configId); - client.submit(j2, &configId); + client.submit(j1, configId); + client.submit(j2, configId); client.waitForJob(j1); client.waitForJob(j2); Index: modules/provider-coaster-c-client/src/RunCoasterJob.cpp =================================================================== --- modules/provider-coaster-c-client/src/RunCoasterJob.cpp (revision 4016) +++ modules/provider-coaster-c-client/src/RunCoasterJob.cpp (working copy) @@ -258,7 +258,7 @@ j.setStderrLocation(*str); } - client.submit(j, &configId); + client.submit(j, configId); client.waitForJob(j); Index: modules/provider-coaster-c-client/src/coasters.cpp =================================================================== --- modules/provider-coaster-c-client/src/coasters.cpp (revision 4016) +++ modules/provider-coaster-c-client/src/coasters.cpp (working copy) @@ -103,7 +103,7 @@ try { client->client.stop(); client->loop.stop(); - + delete client; return COASTER_SUCCESS; } catch (const CoasterError& err) { @@ -118,7 +118,7 @@ try { *settings = new Settings(); COASTER_CHECK_MALLOC(*settings); - + return COASTER_SUCCESS; } catch (const CoasterError& err) { return coaster_error_rc(err); @@ -139,7 +139,7 @@ COASTER_CONDITION(str != NULL, COASTER_ERROR_INVALID, "Null Coaster settings string"); string key, value; // Storage for current key and value - + bool in_key = true; // Either in key or value bool in_quotes = false; bool in_escape = false; @@ -174,9 +174,9 @@ } else if (c == separator) { COASTER_CONDITION(!in_key, COASTER_ERROR_INVALID, "',' not allowed in unquoted Coaster settings key"); - - settings_emit(settings, key, value); - + + settings_emit(settings, key, value); + in_key = true; } else { curr.push_back(c); @@ -207,7 +207,7 @@ string old_value; settings->get(key, old_value); LogWarn << "Overwrote previous Coaster settings value for " - "key: \"" << key << "\". Old value: \"" << + "key: \"" << key << "\". Old value: \"" << old_value << "\", New value: \"" << value << "\"." << endl; } @@ -222,7 +222,7 @@ const char *key, size_t key_len, const char *value, size_t value_len) COASTER_THROWS_NOTHING { try { - settings->set(key, key_len, value, value_len); + settings->set(key, key_len, value, value_len); return COASTER_SUCCESS; } catch (const CoasterError& err) { return coaster_error_rc(err); @@ -259,7 +259,7 @@ // Use malloc so C client code can free *keys = (const char**)malloc(sizeof((*keys)[0]) * (*count)); COASTER_CHECK_MALLOC(*keys); - + if (key_lens != NULL) { *key_lens = (size_t *)malloc(sizeof((*key_lens)[0]) * (*count)); if (!(*key_lens)) { @@ -267,7 +267,7 @@ COASTER_CHECK_MALLOC(*key_lens); } } - + int pos = 0; for(std::map::iterator iter = map.begin(); iter != map.end(); ++iter) { @@ -331,7 +331,7 @@ try { assert(executable != NULL); Job *j = new Job(string(executable, executable_len)); - + for (int i = 0; i < argc; i++) { assert(argv[i] != NULL); @@ -390,7 +390,7 @@ if (job == NULL) { return coaster_return_error(COASTER_ERROR_INVALID, "invalid arg"); } - + try { // job expects to get ownership of references, so use new if (stdin_loc != NULL) { @@ -419,7 +419,7 @@ if (job == NULL) { return coaster_return_error(COASTER_ERROR_INVALID, ""); } - + try { if (dir != NULL) { job->setDirectory(*new string(dir, dir_len)); @@ -440,7 +440,7 @@ if (job == NULL) { return coaster_return_error(COASTER_ERROR_INVALID, "invalid arg"); } - + try { for (int i = 0; i < nvars; i++) { @@ -469,11 +469,11 @@ coaster_job_set_attrs(coaster_job *job, int nattrs, const char **names, size_t *name_lens, const char **values, size_t *value_lens) COASTER_THROWS_NOTHING { - + if (job == NULL) { return coaster_return_error(COASTER_ERROR_INVALID, "invalid job"); } - + try { for (int i = 0; i < nattrs; i++) { @@ -498,11 +498,11 @@ coaster_job_add_cleanups(coaster_job *job, int ncleanups, const char **cleanups, size_t *cleanup_lens) COASTER_THROWS_NOTHING { - + if (job == NULL) { return coaster_return_error(COASTER_ERROR_INVALID, "invalid job"); } - + try { for (int i = 0; i < ncleanups; i++) { const char *cleanup = cleanups[i]; @@ -529,14 +529,14 @@ if (job == NULL) { return coaster_return_error(COASTER_ERROR_INVALID, "invalid job"); } - + try { for (int i = 0; i < nstageins; i++) { coaster_stage_entry *s = &stageins[i]; job->addStageIn(string(s->src, s->src_len), string(s->dst, s->dst_len), s->mode); } - + for (int i = 0; i < nstageouts; i++) { coaster_stage_entry *s = &stageouts[i]; job->addStageOut(string(s->src, s->src_len), @@ -566,7 +566,7 @@ return coaster_return_error(COASTER_ERROR_INVALID, "invalid or unsubmitted job"); } - + *code = status->getStatusCode(); return COASTER_SUCCESS; } @@ -605,8 +605,15 @@ coaster_rc coaster_submit(coaster_client *client, const coaster_config_id *config, coaster_job *job) COASTER_THROWS_NOTHING { + if (client == NULL) { + return coaster_return_error(COASTER_ERROR_INVALID, "invalid client"); + } + if (config == NULL) { + return coaster_return_error(COASTER_ERROR_INVALID, "invalid config"); + } + try { - client->client.submit(*job, config); + client->client.submit(*job, *config); return COASTER_SUCCESS; } catch (const CoasterError& err) { return coaster_error_rc(err); @@ -622,12 +629,12 @@ if (client == NULL) { return coaster_return_error(COASTER_ERROR_INVALID, "invalid client"); } - + try { if (wait) { client->client.waitForAnyJob(); } - + int n = client->client.getAndPurgeDoneJobs(maxjobs, jobs); *njobs = n; @@ -689,7 +696,7 @@ if (prev != NULL) { cleanup_err_key(prev); } - + coaster_err_info *err_info = new coaster_err_info(msg); pthread_setspecific(err_key, static_cast(err_info)); } @@ -699,7 +706,7 @@ if (prev != NULL) { cleanup_err_key(prev); } - + pthread_setspecific(err_key, NULL); } @@ -708,7 +715,7 @@ */ static coaster_rc coaster_return_error(coaster_rc code, const string& msg) { - set_err_info(msg); + set_err_info(msg); return code; } @@ -738,7 +745,7 @@ } coaster_rc coaster_set_log_threshold(coaster_log_level threshold) { - + Logger::singleton().setThreshold(threshold); return COASTER_SUCCESS; Index: modules/provider-coaster-c-client/src/JobSubmitCommand.h =================================================================== --- modules/provider-coaster-c-client/src/JobSubmitCommand.h (revision 4016) +++ modules/provider-coaster-c-client/src/JobSubmitCommand.h (working copy) @@ -13,11 +13,10 @@ private: Job* job; std::string ss; - bool haveConfigId; std::string configId; public: static std::string NAME; - JobSubmitCommand(Job* job, const std::string* configId); + JobSubmitCommand(Job* job, const std::string& configId); virtual void send(CoasterChannel* channel, CommandCallback* cb); Job* getJob(); std::string getRemoteId(); Index: modules/provider-coaster-c-client/src/CoasterClient.h =================================================================== --- modules/provider-coaster-c-client/src/CoasterClient.h (revision 4016) +++ modules/provider-coaster-c-client/src/CoasterClient.h (working copy) @@ -75,9 +75,9 @@ * with the caller, but this client will retain a reference * to the job until done jobs are purged. * - * configId: if non-null, uses specific server-side config + * configId: service-side config to use */ - void submit(Job& job, const std::string* configId); + void submit(Job& job, const std::string& configId); /* * Wait for job to be done. Upon completion no actions Index: modules/provider-coaster-c-client/src/coasters.h =================================================================== --- modules/provider-coaster-c-client/src/coasters.h (revision 4016) +++ modules/provider-coaster-c-client/src/coasters.h (working copy) @@ -313,6 +313,8 @@ * Ownership of the job is shared between the caller and * the client until the job has completed, or the client * shuts down. + * + * config: required, must be non-null */ coaster_rc coaster_submit(coaster_client *client, const coaster_config_id *config, Index: modules/provider-coaster-c-client/src/JobSubmitCommand.cpp =================================================================== --- modules/provider-coaster-c-client/src/JobSubmitCommand.cpp (revision 4016) +++ modules/provider-coaster-c-client/src/JobSubmitCommand.cpp (working copy) @@ -19,15 +19,9 @@ string JobSubmitCommand::NAME("SUBMITJOB"); -JobSubmitCommand::JobSubmitCommand(Job* pjob, const std::string* pconfigId): Command(&NAME) { +JobSubmitCommand::JobSubmitCommand(Job* pjob, const std::string& pconfigId): Command(&NAME) { job = pjob; - if (pconfigId != NULL) { - haveConfigId = true; - configId = *pconfigId; - } - else { - haveConfigId = false; - } + configId = pconfigId; } void JobSubmitCommand::send(CoasterChannel* channel, CommandCallback* cb) { @@ -51,9 +45,7 @@ void JobSubmitCommand::serialize() { stringstream idSS; idSS << job->getIdentity(); - if (haveConfigId) { - add(ss, "configid", configId); - } + add(ss, "configid", configId); add(ss, "identity", idSS.str()); add(ss, "executable", job->getExecutable()); add(ss, "directory", job->getDirectory()); From hategan at ci.uchicago.edu Wed Jul 9 14:37:52 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 9 Jul 2014 14:37:52 -0500 (CDT) Subject: [Swift-commit] r7974 - trunk/etc Message-ID: <20140709193752.B2A739D7A2@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-09 14:37:52 -0500 (Wed, 09 Jul 2014) New Revision: 7974 Added: trunk/etc/swift.conf Removed: trunk/etc/sites.xml trunk/etc/swift.properties trunk/etc/tc.data Log: new config mechanism I Deleted: trunk/etc/sites.xml =================================================================== --- trunk/etc/sites.xml 2014-07-04 18:29:09 UTC (rev 7973) +++ trunk/etc/sites.xml 2014-07-09 19:37:52 UTC (rev 7974) @@ -1,27 +0,0 @@ - - - - - - - - - /var/tmp - 0 - - - Added: trunk/etc/swift.conf =================================================================== --- trunk/etc/swift.conf (rev 0) +++ trunk/etc/swift.conf 2014-07-09 19:37:52 UTC (rev 7974) @@ -0,0 +1,236 @@ +# +# The host name of the submit machine is used by GRAM as a callback +# address to report the status of submitted jobs. In general, Swift +# can automatically detect the host name of the local machine. +# However, if the machine host name is improperly configured or if +# it does not represent a valid DNS entry, certain services (such as +# GRAM) will not be able to send job status notifications back to +# the client. The value of this property can be an IP address. +# +# Format: +# hostName=string +# + + +#hostName=localhost + +# +# A TCP port range can be specified to restrict the ports on which GRAM +# callback services are started. This is likely needed if your submit +# host is behind a firewall, in which case the firewall should be +# configured to allow incoming connections on ports in the range. +# +# Format: +# tcp.port.range=start,end +# + +TCPPortRange="50000, 50100" + +# +# false - means an error will be immediately reported and cause the +# workflow to abort. At this time remote jobs that are already +# running will not be canceled +# true - means that Swift will try to do as much work as possible and +# report all errors encountered at the end. However, "errors" +# here only applies to job execution errors. Certain errors +# that are related to the Swift implementation (should such +# errors occur) will still be reported eagerly. +# +# Default: false +# +lazyErrors=true + +# +# Enables tracing of procedure invocations, assignments, +# iteration constructs, as well as certain dataflow events +# such as data intialization and waiting. This is done at +# a slight decrease in performance. Traces will be available +# in the log file. +# + +tracingEnabled=false + +# +# Indicates when wrapper logs should be fetched from the remote site: +# true - always transfer wrapper logs +# false - only transfer wrapper logs if the job fails +# +# Default: false +# + +alwaysTransferWrapperLog=true + +########################################################################### +# Throttling options # +########################################################################### +# +# For the throttling parameters, valid values are either a positive integer +# or "off" (without the quotes). +# + +# +# Limits the number of concurrent submissions for a workflow instance. This +# throttle only limits the number of concurrent tasks (jobs) that are being +# sent to sites, not the total number of concurrent jobs that can be run. +# The submission stage in GRAM is one of the most CPU expensive stages (due +# mostly to the mutual authentication and delegation). Having too many +# concurrent submissions can overload either or both the submit host CPU +# and the remote host/head node causing degraded performance. +# +# Default: 4 +# + +jobSubmitThrottle=4 +#jobSubmitThrottle=off + +# +# Limits the number of concurrent submissions for any of the sites Swift will +# try to send jobs to. In other words it guarantees that no more than the +# value of this throttle jobs sent to any site will be concurrently in a state +# of being submitted. +# +# Default: 2 +# + +hostJobSubmitThrottle=2 +#hostJobSubmitThrottle=off + +# +# Limits the total number of concurrent file transfers that can happen at any +# given time. File transfers consume bandwidth. Too many concurrent transfers +# can cause the network to be overloaded preventing various other signalling +# traffic from flowing properly. +# +# Default: 4 +# + +fileTransfersThrottle=4 +#fileTransfersThrottle=off + +# Limits the total number of concurrent file operations that can happen at any +# given time. File operations (like transfers) require an exclusive connection +# to a site. These connections can be expensive to establish. A large number +# of concurrent file operations may cause Swift to attempt to establish many +# such expensive connections to various sites. Limiting the number of concurrent +# file operations causes Swift to use a small number of cached connections and +# achieve better overall performance. +# +# Default: 8 +# + +fileOperationsThrottle=8 +#fileOperationsThrottle=off + +# Indicates whether the working directory on the remote site should be +# left intact even when the workflow completes successfully. This can be +# used to inspect the site working directory for debugging purposes. +# +# Default: false +# + +keepSiteDir=false + +# number of time a job will be retried if it fails (giving a maximum of +# 1 + execution.retries attempts at execution) +# + +executionRetries=0 + + +# Enables/disables replication. Replication is used to deal with jobs sitting +# in batch queues for abnormally large amounts of time. If replication is enabled +# and certain conditions are met, Swift creates and submits replicas of jobs, and +# allows multiple instances of a job to compete. +# + +replicationEnabled=false + +# If replication is enabled, this value specifies the minimum time, in seconds, +# a job needs to be queued in a batch queue in order to be considered for +# replication +# + +replicationMinQueueTime=60 + +# The maximum number of replicas that Swift should attempt. + +replicationLimit=3 + +# Controls how Swift will communicate the result code of running user programs +# from workers to the submit side. In files mode, a file +# indicating success or failure will be created on the site shared filesystem. +# In provider mode, the execution provider job status will +# be used. Notably, GRAM2 does not return job statuses correctly, and so +# provider mode will not work with GRAM2. With other +# providers, it can be used to reduce the amount of filesystem access compared +# to files mode. +# +# statusMode=files + +# Controls how swift will supply parameters to the remote wrapper script. +# 'args' mode will pass parameters on the command line +# 'files' mode will pass parameters through an additional input file +# +# valid values: args, files +# Default: files +# +# wrapperParameterMode=args + +# Determines if Swift remote wrappers will be executed by specifying an +# absolute path, or a path relative to the job initial working directory +# +# valid values: absolute, relative +# wrapperInvocationMode=absolute + +# +# Limits the number of concurrent iterations that each foreach statement +# can have at one time. This conserves memory for swift programs that +# have large numbers of iterations (which would otherwise all be executed +# in parallel). +# +# Default: 16384 +# + +maxForeachThreads=16384 + +# controls whether the log file will contain provenance information +# enabling this will increase the size of log files, sometimes +# significantly. + +logProvenance=false + +# The URL prefix used to access local files in wrapper +# staging mode. Full URLs are created by concatenating: +# * the value of this property +# * the current swift work directory (as an absolute path) +# * the file in question (relative to the current work directory) +# +#wrapperStagingLocalServer=file:// + +# Files mapped by the concurrent mapper (i.e. when you don't +# explicitly specify a mapper) are deleted when they are not +# in use any more. This property can be used to prevent +# files mapped by the concurrent mapper from being deleted. +# +# Format: +# fileGCEnabled=(true|false) +# +# Default: true +# + +site.local { + execution { + type: local + } + filesystem { + type: local + } + + workDirectory: "." + + app.ALL { + executable: "*" + } +} + +sites: [local] \ No newline at end of file Deleted: trunk/etc/swift.properties =================================================================== --- trunk/etc/swift.properties 2014-07-04 18:29:09 UTC (rev 7973) +++ trunk/etc/swift.properties 2014-07-09 19:37:52 UTC (rev 7974) @@ -1,388 +0,0 @@ -sites.file=${swift.home}/etc/sites.xml -tc.file=${swift.home}/etc/tc.data - -# -# The host name of the submit machine is used by GRAM as a callback -# address to report the status of submitted jobs. In general, Swift -# can automatically detect the host name of the local machine. -# However, if the machine host name is improperly configured or if -# it does not represent a valid DNS entry, certain services (such as -# GRAM) will not be able to send job status notifications back to -# the client. The value of this property can be an IP address. -# -# Format: -# hostname=string -# - - -#hostname=localhost - -# -# A TCP port range can be specified to restrict the ports on which GRAM -# callback services are started. This is likely needed if your submit -# host is behind a firewall, in which case the firewall should be -# configured to allow incoming connections on ports in the range. -# -# Format: -# tcp.port.range=start,end -# - -#tcp.port.range=50000,50100 - -# -# false - means an error will be immediately reported and cause the -# workflow to abort. At this time remote jobs that are already -# running will not be canceled -# true - means that Swift will try to do as much work as possible and -# report all errors encountered at the end. However, "errors" -# here only applies to job execution errors. Certain errors -# that are related to the Swift implementation (should such -# errors occur) will still be reported eagerly. -# -# Default: false -# -lazy.errors=false - -# -# Enables tracing of procedure invocations, assignments, -# iteration constructs, as well as certain dataflow events -# such as data intialization and waiting. This is done at -# a slight decrease in performance. Traces will be available -# in the log file. -# - -tracing.enabled=true - -# -# What algorithm to use for caching of remote files. LRU (as in what -# files to purge) is the only implementation right now. One can set -# a target size (in bytes) for a host by using the swift:storagesize -# profile for a host in sites.xml -# -# Default: LRU -# -caching.algorithm=LRU - -# -# true - generate a provenance graph in .dot format (Swift will -# choose a random file name) -# false - do not generate a provenance graph -# - generate a provenange graph in the give file name -# -# Default: false -# -pgraph=false - - -# -# graph properties for the provenance graph (.dot specific) -# -# Default: splines="compound", rankdir="TB" -# -pgraph.graph.options=splines="compound", rankdir="TB" - - -# -# node properties for the provenance graph (.dot specific) -# -# Default: color="seagreen", style="filled" -# -pgraph.node.options=color="seagreen", style="filled" - -# -# true - clustering of small jobs is enabled. Clustering works in the -# following way: If a job is clusterable (meaning that it has the -# GLOBUS::maxwalltime profile specified in tc.data and its value -# is less than the value of the "clustering.min.time" property) it will -# be put in a clustering queue. The queue is processed at intervals -# specified by the "clustering.queue.delay" property. The processing -# of the clustering queue consists of selecting compatible jobs and -# grouping them in clusters whose max wall time does not exceed twice -# the value of the "clustering.min.time" property. Two or more jobs are -# considered compatible if they share the same site and do not have -# conflicting profiles (e.g. different values for the same environment -# variable). -# false - clustering of small jobs is disabled. -# -# Default: false -# -clustering.enabled=false - - -# -# - the intervals at which the clustering queue is processed -# -# Default: 4 -# -clustering.queue.delay=4 - -# -# - the threshold time for clustering -# -# Default: 60 -# -clustering.min.time=60 - -# -# Kickstart is a useful tool that can be used to gather various information -# about a remote process. Before it can be used it must be installed on the -# remote site and the corresponding entry be set in the sites file. -# This option allows controlling of how Swift uses Kickstart. The following -# values are possible: -# false - do not use Kickstart -# true - use Kickstart. If a job is scheduled on a site that does not have -# Kickstart installed, that job will fail. -# maybe - Use Kickstart if installed (i.e. the entry is present in the sites -# file) -# -# Default: maybe -# - -kickstart.enabled=maybe - -# -# Indicates when Kickstart records should be fetched from the remote site: -# true - always transfer Kickstart records if Kickstart was used (see -# kickstart.enabled) -# false - only transfer Kickstart records if the job fails -# -# Default: false -# - -kickstart.always.transfer=false - -# -# Indicates when wrapper logs should be fetched from the remote site: -# true - always transfer wrapper logs -# false - only transfer wrapper logs if the job fails -# -# Default: false -# - -wrapperlog.always.transfer=false - -########################################################################### -# Throttling options # -########################################################################### -# -# For the throttling parameters, valid values are either a positive integer -# or "off" (without the quotes). -# - -# -# Limits the number of concurrent submissions for a workflow instance. This -# throttle only limits the number of concurrent tasks (jobs) that are being -# sent to sites, not the total number of concurrent jobs that can be run. -# The submission stage in GRAM is one of the most CPU expensive stages (due -# mostly to the mutual authentication and delegation). Having too many -# concurrent submissions can overload either or both the submit host CPU -# and the remote host/head node causing degraded performance. -# -# Default: 4 -# - -throttle.submit=4 -#throttle.submit=off - -# -# Limits the number of concurrent submissions for any of the sites Swift will -# try to send jobs to. In other words it guarantees that no more than the -# value of this throttle jobs sent to any site will be concurrently in a state -# of being submitted. -# -# Default: 2 -# - -throttle.host.submit=2 -#throttle.host.submit=off - -# -# The Swift scheduler has the ability to limit the number of concurrent jobs -# allowed on a site based on the performance history of that site. Each site -# is assigned a score (initially 1), which can increase or decrease based -# on whether the site yields successful or faulty job runs. The score for a -# site can take values in the (0.1, 100) interval. The number of allowed jobs -# is calculated using the following formula: -# 2 + score*throttle.score.job.factor -# This means a site will always be allowed at least two concurrent jobs and -# at most 2 + 100*throttle.score.job.factor. With a default of 4 this means -# at least 2 jobs and at most 402. -# -# Default: 4 -# - -throttle.score.job.factor=0.2 -#throttle.score.job.factor=off - - -# -# Limits the total number of concurrent file transfers that can happen at any -# given time. File transfers consume bandwidth. Too many concurrent transfers -# can cause the network to be overloaded preventing various other signalling -# traffic from flowing properly. -# -# Default: 4 -# - -throttle.transfers=4 -#throttle.transfers=off - -# Limits the total number of concurrent file operations that can happen at any -# given time. File operations (like transfers) require an exclusive connection -# to a site. These connections can be expensive to establish. A large number -# of concurrent file operations may cause Swift to attempt to establish many -# such expensive connections to various sites. Limiting the number of concurrent -# file operations causes Swift to use a small number of cached connections and -# achieve better overall performance. -# -# Default: 8 -# - -throttle.file.operations=8 -#throttle.file.operations=off - -# Indicates whether the working directory on the remote site should be -# left intact even when the workflow completes successfully. This can be -# used to inspect the site working directory for debugging purposes. -# -# Default: false -# - -sitedir.keep=false - -# number of time a job will be retried if it fails (giving a maximum of -# 1 + execution.retries attempts at execution) -# - -execution.retries=0 - - -# Enables/disables replication. Replication is used to deal with jobs sitting -# in batch queues for abnormally large amounts of time. If replication is enabled -# and certain conditions are met, Swift creates and submits replicas of jobs, and -# allows multiple instances of a job to compete. -# - -replication.enabled=false - -# If replication is enabled, this value specifies the minimum time, in seconds, -# a job needs to be queued in a batch queue in order to be considered for -# replication -# - -replication.min.queue.time=60 - -# The maximum number of replicas that Swift should attempt. - -replication.limit=3 - -# -# WARNING: This option is deprecated. Please use the hostname option. -# -# The IP address of the submit machine is used by GRAM as a callback -# address to report the status of submitted jobs. In general, Swift -# can automatically detect the IP address of the local machine. -# However, if the machine has more than one network interface, Swift -# will pick the first one, which may not be the right choice. It is -# recommended that this property is set properly before attempting to -# run jobs through GRAM. -# -# Format: -# ip.address=x.y.z.w -# - -#ip.address=127.0.0.1 - - -# Controls how Swift will communicate the result code of running user programs -# from workers to the submit side. In files mode, a file -# indicating success or failure will be created on the site shared filesystem. -# In provider mode, the execution provider job status will -# be used. Notably, GRAM2 does not return job statuses correctly, and so -# provider mode will not work with GRAM2. With other -# providers, it can be used to reduce the amount of filesystem access compared -# to files mode. -# -# status.mode=files - -# Controls how swift will supply parameters to the remote wrapper script. -# 'args' mode will pass parameters on the command line -# 'files' mode will pass parameters through an additional input file -# -# valid values: args, files -# Default: files -# -# wrapper.parameter.mode=args - -# Determines if Swift remote wrappers will be executed by specifying an -# absolute path, or a path relative to the job initial working directory -# -# valid values: absolute, relative -# wrapper.invocation.mode=absolute - -# -# Limits the number of concurrent iterations that each foreach statement -# can have at one time. This conserves memory for swift programs that -# have large numbers of iterations (which would otherwise all be executed -# in parallel). -# -# Default: 1024 -# - -foreach.max.threads=16384 - -# controls whether the log file will contain provenance information -# enabling this will increase the size of log files, sometimes -# significantly. - -provenance.log=false - -# Controls whether file staging is done by swift or by the execution -# provider. If set to false, the standard swift staging mechanism is -# used. If set to true, swift does not stage files. Instead, the -# execution provider is instructed to stage files in and out. -# -# Provider staging is experimental. -# -# When enabled, and when coasters are used as an execution provider, -# a staging mechanism can be selected for each site -# using the swift:stagingMethod site profile in sites.xml. The -# following is a list of accepted mechanisms: -# -# * file: Staging is done from a filesystem accessible to the -# coaster service (typically running on the head node) -# * proxy: Staging is done from a filesystem accessible to the -# client machine that swift is running on, and is proxied -# through the coaster service -# * sfs: (short for "shared filesystem") Staging is done by -# copying files to and from a filesystem accessible -# by the compute node (such as an NFS or GPFS mount). - - -use.provider.staging=false -provider.staging.pin.swiftfiles=false - -# Controls whether wrapper staging is enabled -# With wrapper staging, files are staged in and out by the -# swift wrapper, on the worker node. - -use.wrapper.staging=false - -# The URL prefix used to access local files in wrapper -# staging mode. Full URLs are created by concatenating: -# * the value of this property -# * the current swift work directory (as an absolute path) -# * the file in question (relative to the current work directory) -# -#wrapper.staging.local.server=file:// - -# Files mapped by the concurrent mapper (i.e. when you don't -# explicitly specify a mapper) are deleted when they are not -# in use any more. This property can be used to prevent -# files mapped by the concurrent mapper from being deleted. -# -# Format: -# file.gc.enabled=(true|false) -# -# Default: true -# Deleted: trunk/etc/tc.data =================================================================== --- trunk/etc/tc.data 2014-07-04 18:29:09 UTC (rev 7973) +++ trunk/etc/tc.data 2014-07-09 19:37:52 UTC (rev 7974) @@ -1 +0,0 @@ -* * * INSTALLED INTEL32::LINUX null From hategan at ci.uchicago.edu Wed Jul 9 14:38:06 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 9 Jul 2014 14:38:06 -0500 (CDT) Subject: [Swift-commit] r7975 - trunk/libexec Message-ID: <20140709193806.624E99D7A2@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-09 14:38:06 -0500 (Wed, 09 Jul 2014) New Revision: 7975 Modified: trunk/libexec/scheduler.k trunk/libexec/swift-int-staging.k trunk/libexec/swift-int-wrapper-staging.k trunk/libexec/swift-int.k trunk/libexec/swift-lib.k trunk/libexec/swift.k Log: new config mechanism II Modified: trunk/libexec/scheduler.k =================================================================== --- trunk/libexec/scheduler.k 2014-07-09 19:37:52 UTC (rev 7974) +++ trunk/libexec/scheduler.k 2014-07-09 19:38:06 UTC (rev 7975) @@ -2,31 +2,20 @@ import(task) import('swift-lib') -sites := swift:configProperty("sites.file") -TCFile := swift:configProperty("tc.file") +config := contextAttribute("SWIFT:CONFIG") -log(LOG:INFO, "Using sites file: {sites}") -if (!file:exists(sites)) { - throw("Could not find sites file: {sites}") -} - -log(LOG:INFO, "Using tc.data: {TCFile}") - -scheduler("vds-adaptive", shareID = "swift:scheduler:{sites}" - property("transformationCatalogFile", TCFile) - property("clusteringEnabled", swift:configProperty("clustering.enabled")) - property("clusteringQueueDelay", swift:configProperty("clustering.queue.delay")) - property("clusteringMinTime", swift:configProperty("clustering.min.time")) +scheduler("vds-adaptive", shareID = "swift:scheduler" + property("config", config) - property("hostSubmitThrottle", swift:configProperty("throttle.host.submit")) - property("submitThrottle", swift:configProperty("throttle.submit")) + property("hostSubmitThrottle", swift:configProperty("hostJobSubmitThrottle")) + property("submitThrottle", swift:configProperty("jobSubmitThrottle")) property("jobsPerCpu", "off") - property("maxTransfers", swift:configProperty("throttle.transfers")) - property("maxFileOperations", swift:configProperty("throttle.file.operations")) - property("jobThrottle", swift:configProperty("throttle.score.job.factor")) + property("maxTransfers", swift:configProperty("fileTransfersThrottle")) + property("maxFileOperations", swift:configProperty("fileOperationsThrottle")) + property("jobThrottle", swift:configProperty("siteScoreThrottlingFactor")) task:availableHandlers(type = "execution", includeAliases = true) task:availableHandlers(type = "file", includeAliases = true) - resources = swift:siteCatalog(sites) + resources = swift:siteCatalog(config) ) Modified: trunk/libexec/swift-int-staging.k =================================================================== --- trunk/libexec/swift-int-staging.k 2014-07-09 19:37:52 UTC (rev 7974) +++ trunk/libexec/swift-int-staging.k 2014-07-09 19:38:06 UTC (rev 7975) @@ -11,19 +11,18 @@ SWIFT:DEBUG_DIR_PREFIX := contextAttribute("SWIFT:DEBUG_DIR_PREFIX") WRAPPER_TRANSFER_MODE := - if (configProperty("wrapperlog.always.transfer") == "true", + if (configProperty("alwaysTransferWrapperLog"), STAGING_MODE:IF_PRESENT, STAGING_MODE:ON_ERROR + STAGING_MODE:IF_PRESENT) -pinOption := configProperty("provider.staging.pin.swiftfiles") +pinOption := configProperty("providerStagingPinSwiftFiles") -PIN := if(pinOption == "true", "pinned:", "") -PROVENANCE_GRAPH_ENABLED := (configProperty("pgraph") != "false") -CLEANUP_ENABLED := (configProperty("sitedir.keep") != "true") +PIN := if(pinOption, "pinned:", "") +CLEANUP_ENABLED := !configProperty("keepSiteDir") DEBUG_DIR := "{SWIFT:DEBUG_DIR_PREFIX}{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d" CDM_FILE := cdm:file() -namespace(swift) { +namespace(providerStaging) { fileSizes := function(files) { math:sum( @@ -31,12 +30,6 @@ ) } - export(cleanups, - function(cleanup) { - log(LOG:INFO, "START cleanups={cleanup}") - } - ) - readErrorFiles := function(dir, jobid, stdout, stderr) { concat( if(file:exists("{dir}/{jobid}.error")) { @@ -59,9 +52,8 @@ } export(execute2, - function(rhost, progress, tr, stagein, stageout, - replicationGroup, replicationChannel, - arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { + function(rhost, progress, tr, arguments, attributes, stdin, stdout, stderr, + stagein, stageout, replicationGroup, replicationChannel) { uid := UID() jobdir := substring(uid, 0, to=1) Modified: trunk/libexec/swift-int-wrapper-staging.k =================================================================== --- trunk/libexec/swift-int-wrapper-staging.k 2014-07-09 19:37:52 UTC (rev 7974) +++ trunk/libexec/swift-int-wrapper-staging.k 2014-07-09 19:38:06 UTC (rev 7975) @@ -2,28 +2,17 @@ import(task) import('swift-lib') +URL_PREFIX := getURLPrefix() -getURLPrefix := def("org.griphyn.vdl.karajan.lib.GetURLPrefix") +WRAPPER_LOG_ALWAYS_TRANSFER := configProperty("alwaysTransferWrapperLog") +SWIFT:SCRIPT_NAME := contextAttribute("SWIFT:SCRIPT_NAME") +SWIFT:RUN_ID := contextAttribute("SWIFT:RUN_ID") +SWIFT:HOME := contextAttribute("SWIFT:HOME") +SITEDIR_KEEP := configProperty("keepSiteDir") -URL_PREFIX := getURLPrefix() -WRAPPERLOG_ALWAYS_TRANSFER := vdl:configProperty("wrapperlog.always.transfer") -SITEDIR_KEEP := vdl:configProperty("sitedir.keep") - - -namespace(swift) { +namespace(wrapperStaging) { - checkJobStatus := function(jobdir, jobid, tr) { - log(LOG:DEBUG, "START jobid={jobid}") - try { - file:remove("{jobdir}/_swift.success") - log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") - } - else { - throw(checkErrorFile(jobdir, jobid)) - } - } - checkErrorFile := function(jobdir, jobid) { if (file:exists("{jobdir}/_swift.error")) { log(LOG:INFO, "FAILURE jobid={jobid} - Failure file found") @@ -36,6 +25,17 @@ throw("No status file was found") } } + + checkJobStatus := function(jobdir, jobid, tr) { + log(LOG:DEBUG, "START jobid={jobid}") + try { + file:remove("{jobdir}/_swift.success") + log(LOG:INFO, "SUCCESS jobid={jobid} - Success file found") + } + else { + throw(checkErrorFile(jobdir, jobid)) + } + } initSharedDir := function(progress, rhost) { once(list(rhost, "shared")) { @@ -43,9 +43,9 @@ log(LOG:INFO, "START host={rhost} - Initializing shared directory") - wfdir := "{VDL:SCRIPTNAME}-{VDL:RUNID}" + wfdir := "{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}" dir:make(wfdir, host=rhost) - transfer(srcdir="{swift.home}/libexec/", srcfile="_swiftwrap.wrapperstaging", destdir=wfdir, desthost=rhost) + transfer(srcdir="{SWIFT:HOME}/libexec/", srcfile="_swiftwrap.wrapperstaging", destdir=wfdir, desthost=rhost) wfdir to(cleanup, list(wfdir, rhost)) @@ -64,48 +64,14 @@ ddir } - cleanup := function(dir, host) { - log(LOG:INFO, "START dir={dir} host={host}") - if(vdl:configProperty("sitedir.keep") == "false") { - task:execute( - vdl:siteprofile(host, "cleanupCommand"), - arguments = list( - siteProfile(host, "cleanupCommandOptions"), - dir - ) - host=host, batch=true, TCProfile(host) - ) - log(LOG:INFO, "END dir={dir} host={host}") - } - } + stageWrapperParams := function(jobid, jobdir, wrapfile, dir, host) { + log(LOG:INFO, "START jobid={jobid} - staging in wrapper params"), + (provider, srchost, destdir, filename, srcdir) := splitFileURL(wrapfile, dir, destdir="parameters/{jobdir}") - cleanups := function(cleanup) { - log(LOG:INFO, "START cleanups={cleanup}") - parallelFor(i, cleanup) { - (dir, host) := each(i) - try { - vdl:cleanup(dir, host) - } - else catch(exception) { - log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception) - to(warnings, exception("Cleanup on {host} failed", exception)) - } + cache(list(destdir, host)) { + dir:make(destdir, host=host, provider=provider) } - log(LOG:INFO, "END cleanups={cleanup}") - } - stageWrapperParams := function(jobid, wrapfile, dir, host) { - log(LOG:INFO, "START jobid={jobid} - staging in wrapper params") - provider := provider(wrapfile) - srchost := hostname(wrapfile) - srcdir := vdl:dirname(wrapfile) - destdir := dir - filename := basename(wrapfile) - - cacheOn(list(destdir, host) - dir:make(destdir, host=host, provider=provider) - ) - log(LOG:INFO, "END jobid={jobid}") } @@ -129,9 +95,8 @@ export(execute2, - function(rhost, progress, tr, stagein, stageout, - replicationGroup, replicationChannel, - arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { + function(rhost, progress, tr, arguments, attributes, stdin, stdout, stderr, + stagein, stageout, replicationGroup, replicationChannel) { ddir := initDDir() wfdir := try { @@ -146,10 +111,10 @@ jobdir := concat(ddir, "/jobs/", substring(uid, from=0, to=1), "/{jobid}/") - log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread={#thread} host={rhost} replicationGroup={replicationGroup}") + log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") - statusMode := configProperty("status.mode", host = rhost) - wrapperMode := configProperty("wrapper.parameter.mode", host = rhost) + statusMode := configProperty("statusMode", host = rhost) + wrapperMode := configProperty("wrapperParameterMode", host = rhost) wrapfile := "{jobdir}/_paramfile" @@ -163,22 +128,22 @@ scratch := siteProfile(rhost, "scratch") if(wrapperMode == "files") { - file:write(wrapfile, - "-e ",vdl:executable(tr, rhost), + file:write(wrapfile) { + "-e ", executable(tr, rhost), "\n-out ", stdout, "\n-err ", stderr, "\n-i ", if (stdin != null, getFieldValue(stdin)), "\n-d ", str:join(remoteFileDirs, "|"), "\n-if ", str:join(remoteFileNames(inFiles), "|"), "\n-of ", str:join(remoteFileNames(outFiles), "|"), - "\n-wt", WRAPPERLOG_ALWAYS_TRANSFER, + "\n-wt", WRAPPER_LOG_ALWAYS_TRANSFER, "\n-sk", SITEDIR_KEEP, "\n-cdmfile ", cdm:file(), "\n-status ", statusMode, for(a, arguments) { "\n-a ", a } - ) + } } @@ -186,7 +151,7 @@ try { if (wrapperMode == "files") { - stageWrapperParams(jobid, wrapfile, wfdir, rhost) + stageWrapperParams(jobid, jobdir, wrapfile, wfdir, rhost) } log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)), " host={rhost}") @@ -225,14 +190,14 @@ "-urlprefix", URL_PREFIX, "-jobdir", jobdir, "-scratch", scratch, - "-e", vdl:executable(tr, rhost), + "-e", executable(tr, rhost), "-out", stdout, "-err", stderr, "-i", if (stdin != null, getFieldValue(stdin)), "-d", str:join(remoteFileDirs, "|"), "-if", str:join(remoteFileNames(inFiles), "|"), "-of", str:join(remoteFileNames(outFiles), "|"), - "-wt", WRAPPERLOG_ALWAYS_TRANSFER, + "-wt", WRAPPER_LOG_ALWAYS_TRANSFER, "-sk", SITEDIR_KEEP, "-cdmfile", cdm:file(), "-status", statusMode, @@ -265,7 +230,7 @@ setProgress(progress, "Stage out") - doRestartlog(stageout) + doRestartLog(stageout) log(LOG:DEBUG, "JOB_END jobid={jobid}") } @@ -292,7 +257,7 @@ "Exception in {tr}:", if (arguments != null, "\n Arguments: {arguments}") "\n Host: {rhost}", - "\n Directory: {tmpdir}", + "\n Directory: {jobdir}", "{outs}", ) exception Modified: trunk/libexec/swift-int.k =================================================================== --- trunk/libexec/swift-int.k 2014-07-09 19:37:52 UTC (rev 7974) +++ trunk/libexec/swift-int.k 2014-07-09 19:38:06 UTC (rev 7975) @@ -14,11 +14,13 @@ SHARED_DIR := dircat(RUN_DIR, "shared") DEBUG_DIR := "{SWIFT:DEBUG_DIR_PREFIX}{SWIFT:SCRIPT_NAME}-{SWIFT:RUN_ID}.d" +WRAPPER_LOG_ALWAYS_TRANSFER := configProperty("alwaysTransferWrapperLog") + if (!file:exists(DEBUG_DIR)) { task:dir:make(DEBUG_DIR) } -namespace(swift) { +namespace(swiftStaging) { rmdir := function(dir, host) { parallelFor(entry, file:list(dir, host=host)) { @@ -86,12 +88,12 @@ transfer(siteProfile(rhost, "wrapperScript"), srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) transfer("_swiftseq", srcdir="{SWIFT:HOME}/libexec/", destdir=SHARED_DIR, desthost=rhost) - statusMode := configProperty("status.mode", host=rhost) + statusMode := configProperty("statusMode", host=rhost) if (statusMode == "files") { dir:make(dircat(RUN_DIR, "status"), host=rhost) } - wrapperMode := configProperty("wrapper.parameter.mode", host=rhost) + wrapperMode := configProperty("wrapperParameterMode", host=rhost) if (wrapperMode == "files") { dir:make(dircat(RUN_DIR, "parameters"), host=rhost) } @@ -121,30 +123,6 @@ log(LOG:INFO, "END jobid={jobid} - Done initializing directory structure") } - cleanup := function(dir, host) { - log(LOG:INFO, "START dir={dir} host={host}") - cdmfile := cdm:file() - log(LOG:DEBUG, "cdmfile {cdmfile}") - if (cdmfile != "" & cdm:get("GATHER_DIR") != "UNSET") { - log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}") - task:transfer("cdm_cleanup.sh", - srcdir="{SWIFT:HOME}/libexec", - desthost=host, destdir=dir) - task:transfer("cdm_lib.sh", - srcdir="{SWIFT:HOME}/libexec", - desthost=host, destdir=dir) - log(LOG:INFO, "execute: cdm_cleanup.sh") - task:execute("/bin/bash", list("{dir}/cdm_cleanup.sh", cdm:get("GATHER_DIR"), cdm:get("GATHER_TARGET"), UID()) - host=host, batch=true, TCProfile(host)) - } - if (swift:configProperty("sitedir.keep") == "false") { - task:execute(siteProfile(host, "cleanupCommand"), - list(siteProfile(host, "cleanupCommandOptions"), dir) - host=host, batch=true, TCProfile(host)) - } - log(LOG:INFO, "END dir={dir} host={host}") - } - cleanupFiles := function(files, host) { parallelFor(r, files) { log(LOG:INFO, "Purging ", r, " on ", host) @@ -313,28 +291,10 @@ } recfile } - - export(cleanups, - function(cleanup) { - log(LOG:INFO, "START cleanups={cleanup}") - parallelFor(i, cleanup) { - (dir, host) := each(i) - try { - cleanup(dir, host) - } - else catch(exception) { - log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception) - to(warnings, exception("Cleanup on {host} failed", exception)) - } - } - log(LOG:INFO, "END cleanups={cleanup}") - } - ) export(execute2, - function(rhost, progress, tr, stagein, stageout, - replicationGroup, replicationChannel, - arguments = [], stdin = null, stdout = null, stderr = null, attributes = null) { + function(rhost, progress, tr, arguments, attributes, stdin, stdout, stderr, + stagein, stageout, replicationGroup, replicationChannel) { try { initSharedDir(progress, rhost) @@ -349,8 +309,8 @@ log(LOG:DEBUG, "THREAD_ASSOCIATION jobid={jobid} thread=", currentThread(), " host={rhost} replicationGroup={replicationGroup}") - statusMode := configProperty("status.mode",host=rhost) - wrapperMode := configProperty("wrapper.parameter.mode",host=rhost) + statusMode := configProperty("statusMode", host=rhost) + wrapperMode := configProperty("wrapperParameterMode", host=rhost) wrapfile := "{DEBUG_DIR}/param-{jobid}" @@ -469,7 +429,7 @@ doStageoutCollect(jobid, SHARED_DIR, rhost, outFiles) } - if (configProperty("wrapperlog.always.transfer") == "true") { + if (WRAPPER_LOG_ALWAYS_TRANSFER) { discard(transferWrapperLog(rhost, jobid, jobdir)) } Modified: trunk/libexec/swift-lib.k =================================================================== --- trunk/libexec/swift-lib.k 2014-07-09 19:37:52 UTC (rev 7974) +++ trunk/libexec/swift-lib.k 2014-07-09 19:38:06 UTC (rev 7975) @@ -139,6 +139,8 @@ export(unwrapClosedList, def("org.griphyn.vdl.karajan.lib.UnwrapClosedList")) export(siteCatalog, def("org.griphyn.vdl.karajan.lib.SiteCatalog")) + + export(getURLPrefix, def("org.griphyn.vdl.karajan.lib.GetURLPrefix")) } namespace(cdm) { Modified: trunk/libexec/swift.k =================================================================== --- trunk/libexec/swift.k 2014-07-09 19:37:52 UTC (rev 7974) +++ trunk/libexec/swift.k 2014-07-09 19:38:06 UTC (rev 7975) @@ -5,33 +5,69 @@ import('swift-lib', export = true) import('swift-xs', export = true) +import('swift-int') +import('swift-int-staging') +import('swift-int-wrapper-staging') + SWIFT:SCRIPT_NAME := contextAttribute("SWIFT:SCRIPT_NAME") SWIFT:RUN_ID := contextAttribute("SWIFT:RUN_ID") SWIFT:HOME := contextAttribute("SWIFT:HOME") +SITEDIR_KEEP := configProperty("keepSiteDir") -namespace(swift) { - - pstaging := (configProperty("use.provider.staging") == "true") - wstaging := (configProperty("use.wrapper.staging") == "true") - - impl := if (pstaging, "swift-int-staging.k", if (wstaging, "swift-int-wrapper-staging.k", "swift-int.k")) - - import(file = impl) - - import(java) - +namespace(swift) { export(stageIn, def("org.griphyn.vdl.karajan.lib.Stagein")) export(stageOut, def("org.griphyn.vdl.karajan.lib.Stageout")) export(parameterLog, function(direction, variable, id) { - if (configProperty("provenance.log") == "true") { + if (configProperty("logProvenance")) { thread := currentThread() log("info", "PARAM thread={thread} direction={direction} variable={variable} provenanceid={id}") } } ) + cleanup := function(dir, host) { + log(LOG:INFO, "START dir={dir} host={host}") + cdmfile := cdm:file() + log(LOG:DEBUG, "cdmfile {cdmfile}") + if (cdmfile != "" & cdm:get("GATHER_DIR") != "UNSET") { + log(LOG:INFO, "submitting cdm_cleanup.sh to {dir}") + task:transfer("cdm_cleanup.sh", + srcdir="{SWIFT:HOME}/libexec", + desthost=host, destdir=dir) + task:transfer("cdm_lib.sh", + srcdir="{SWIFT:HOME}/libexec", + desthost=host, destdir=dir) + log(LOG:INFO, "execute: cdm_cleanup.sh") + task:execute("/bin/bash", list("{dir}/cdm_cleanup.sh", cdm:get("GATHER_DIR"), cdm:get("GATHER_TARGET"), UID()) + host=host, batch=true, TCProfile(host)) + } + if (!SITEDIR_KEEP) { + task:execute(siteProfile(host, "cleanupCommand"), + list(siteProfile(host, "cleanupCommandOptions"), dir) + host=host, batch=true, TCProfile(host)) + } + log(LOG:INFO, "END dir={dir} host={host}") + } + + export(cleanups, + function(cleanup) { + log(LOG:INFO, "START cleanups={cleanup}") + parallelFor(i, cleanup) { + (dir, host) := each(i) + try { + cleanup(dir, host) + } + else catch(exception) { + log(LOG:DEBUG, "EXCEPTION - Exception caught while cleaning up", exception) + to(warnings, exception("Cleanup on {host} failed", exception)) + } + } + log(LOG:INFO, "END cleanups={cleanup}") + } + ) + export(split, function(var) { each(str:split(getFieldValue(var), " ")) @@ -92,8 +128,27 @@ to(cleanup, unique(for(c, cleanup, c))) ) } - ) + ) + executeSelect := function(rhost, progress, tr, arguments, attributes, + stdin, stdout, stderr, stagein, stageout, replicationGroup, replicationChannel) { + + staging := configProperty("staging", host = rhost) + + if (staging == "swift") { + swiftStaging:execute2(rhost, progress, tr, arguments, attributes, stdin, stdout, stderr, + stagein, stageout, replicationGroup, replicationChannel) + } + else if (staging == "provider") { + providerStaging:execute2(rhost, progress, tr, arguments, attributes, stdin, stdout, stderr, + stagein, stageout, replicationGroup, replicationChannel) + } + else { + wrapperStaging:execute2(rhost, progress, tr, arguments, attributes, stdin, stdout, stderr, + stagein, stageout, replicationGroup, replicationChannel) + } + } + export(execute, function( tr, arguments = null, @@ -113,8 +168,8 @@ try { throttled { setProgress(progress, "Selecting site") - collectList := restartOnError(number(swift:configProperty("execution.retries"))) { - if (swift:configProperty("replication.enabled") == "true") { + collectList := restartOnError(swift:configProperty("executionRetries")) { + if (swift:configProperty("replicationEnabled")) { replicationChannel := channel:new() //trigger the first job discard(append(replicationChannel, true)) @@ -122,10 +177,11 @@ parallelFor(i, replicationChannel) { try { allocateHost(rhost, constraints = jobConstraints(tr, stagein = stagein)) { - execute2( + executeSelect( rhost, progress, - tr, if(arguments != null, arguments = unwrapClosedList(arguments)), - stdin=stdin, stdout=stdout, stderr=stderr, attributes=attributes, + tr, if(arguments != null, unwrapClosedList(arguments), []), + attributes, + stdin, stdout, stderr, stagein, stageout, replicationGroup, replicationChannel ) } @@ -143,10 +199,11 @@ else { try { allocateHost(rhost, constraints = jobConstraints(tr, stagein = stagein)) { - execute2( + executeSelect( rhost, progress, - tr, if(arguments != null, arguments = unwrapClosedList(arguments)), - stdin=stdin, stdout=stdout, stderr=stderr, attributes=attributes, + tr, if(arguments != null, unwrapClosedList(arguments), []), + attributes, + stdin, stdout, stderr, stagein, stageout, null, null ) } @@ -169,7 +226,7 @@ else catch(exception) { log(LOG:INFO, "END_FAILURE thread=", currentThread(), " tr={tr}") setProgress(progress, "Failed") - if(swift:configProperty("lazy.errors") == "false") { + if(!swift:configProperty("lazyErrors")) { throw(exception) } else { From hategan at ci.uchicago.edu Wed Jul 9 14:40:34 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 9 Jul 2014 14:40:34 -0500 (CDT) Subject: [Swift-commit] r7977 - trunk Message-ID: <20140709194034.9B8699D7A2@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-09 14:40:34 -0500 (Wed, 09 Jul 2014) New Revision: 7977 Modified: trunk/launchers.xml Log: added launchers for swift-convert-config and swift-config-info Modified: trunk/launchers.xml =================================================================== --- trunk/launchers.xml 2014-07-09 19:38:50 UTC (rev 7976) +++ trunk/launchers.xml 2014-07-09 19:40:34 UTC (rev 7977) @@ -12,6 +12,14 @@ + + + + + + + + From hategan at ci.uchicago.edu Wed Jul 9 14:38:50 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 9 Jul 2014 14:38:50 -0500 (CDT) Subject: [Swift-commit] r7976 - in trunk/src/org: globus/swift/catalog/site globus/swift/catalog/types globus/swift/data griphyn/vdl/karajan griphyn/vdl/karajan/functions griphyn/vdl/karajan/lib griphyn/vdl/karajan/lib/replication griphyn/vdl/karajan/lib/swiftscript griphyn/vdl/mapping griphyn/vdl/mapping/file griphyn/vdl/mapping/nodes griphyn/vdl/util Message-ID: <20140709193850.90FB99D7A2@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-09 14:38:50 -0500 (Wed, 09 Jul 2014) New Revision: 7976 Added: trunk/src/org/griphyn/vdl/util/ConfigTree.java trunk/src/org/griphyn/vdl/util/ConvertConfig.java trunk/src/org/griphyn/vdl/util/SwiftConfig.java trunk/src/org/griphyn/vdl/util/SwiftConfigException.java trunk/src/org/griphyn/vdl/util/SwiftConfigInfo.java trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java Removed: trunk/src/org/griphyn/vdl/util/TriStateBoolean.java trunk/src/org/griphyn/vdl/util/VDL2Config.java trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java Modified: trunk/src/org/globus/swift/catalog/site/Application.java trunk/src/org/globus/swift/catalog/site/SwiftContact.java trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java trunk/src/org/globus/swift/catalog/types/Arch.java trunk/src/org/globus/swift/catalog/types/Os.java trunk/src/org/globus/swift/catalog/types/SysInfo.java trunk/src/org/globus/swift/data/Director.java trunk/src/org/griphyn/vdl/karajan/Loader.java trunk/src/org/griphyn/vdl/karajan/SwiftRootScope.java trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java trunk/src/org/griphyn/vdl/karajan/lib/CacheFunction.java trunk/src/org/griphyn/vdl/karajan/lib/Execute.java trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java trunk/src/org/griphyn/vdl/karajan/lib/New.java trunk/src/org/griphyn/vdl/karajan/lib/Operators.java trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java trunk/src/org/griphyn/vdl/karajan/lib/Throttled.java trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java Log: new config mechanism III Modified: trunk/src/org/globus/swift/catalog/site/Application.java =================================================================== --- trunk/src/org/globus/swift/catalog/site/Application.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/globus/swift/catalog/site/Application.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -42,7 +42,7 @@ env.put(name, value); } - public void addProperty(String name, String value) { + public void addProperty(String name, Object value) { if (properties == null) { properties = new HashMap(); } Modified: trunk/src/org/globus/swift/catalog/site/SwiftContact.java =================================================================== --- trunk/src/org/globus/swift/catalog/site/SwiftContact.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/globus/swift/catalog/site/SwiftContact.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -55,16 +55,31 @@ public void setSiteCatalog(SwiftContactSet siteCatalog) { this.siteCatalog = siteCatalog; } + + /** + * Get an application with the specified tr. Only this site + * is searched. Returns null if not found. + */ + public Application getApplication(String tr) { + if (apps == null) { + return null; + } + else { + return apps.get(tr); + } + } + /** + * Find apps by searching in the following order: + *
      + *
    1. host:tr
    2. + *
    3. host:*
    4. + *
    5. pool:tr
    6. + *
    7. pool:*
    8. + *
    + */ public Application findApplication(String tr) { - /* - * Find apps in the following order: - * host:tr - * host:* - * pool:tr - * pool:* - */ - + Application app = null; if (apps != null) { app = apps.get(tr); Modified: trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java =================================================================== --- trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -49,4 +49,11 @@ } return app; } + + public Map getApplications() { + if (apps == null) { + apps = new HashMap(); + } + return apps; + } } Modified: trunk/src/org/globus/swift/catalog/types/Arch.java =================================================================== --- trunk/src/org/globus/swift/catalog/types/Arch.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/globus/swift/catalog/types/Arch.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -17,91 +17,7 @@ package org.globus.swift.catalog.types; -/** - * This is an enumerated data class for the different types of architecture. - * - * @author Gaurang Mehta gmehta at isi.edu - * @version $Revision: 1.5 $ - */ +public enum Arch { + INTEL32, INTEL64, SPARCV7, SPARCV9, AMD64; -import java.io.Serializable; -import java.util.HashMap; - -public class Arch - implements Serializable { - private String _value_; - private static HashMap _table_ = new HashMap(5); - - protected Arch(String value) { - _value_ = value; - _table_.put(_value_, this); - } - - private static final String _INTEL32 = "INTEL32"; - private static final String _INTEL64 = "INTEL64"; - private static final String _SPARCV7 = "SPARCV7"; - private static final String _SPARCV9 = "SPARCV9"; - private static final String _AMD64 = "AMD64"; - - public static final Arch INTEL32 = new Arch(_INTEL32); - public static final Arch INTEL64 = new Arch(_INTEL64); - public static final Arch SPARCV7 = new Arch(_SPARCV7); - public static final Arch SPARCV9 = new Arch(_SPARCV9); - public static final Arch AMD64 = new Arch(_AMD64); - - public static final String err = "Error: Illegal Architecture defined. Please specify one of the predefined types \n [INTEL32, INTEL64, AMD64, SPARCV7, SPARCV9]"; - - /** - * Returns the value of the architecture as string. - * @return String - */ - public String getValue() { - return _value_; - } - - /** - * Creates a new Arch Object givan a arch string. - * @param value String - * @throws IllegalStateException Throws Exception if the architecure is not defined in this class. - * @return Arch - */ - public static Arch fromValue(String value) throws IllegalStateException { - Arch m_enum = _table_.get(value.toUpperCase()); - if (m_enum == null) { - throw new IllegalStateException(err); - } - return m_enum; - } - - /** - * Creates a new Arch object given a arch string. - * @param value String - * @throws IllegalStateException Throws Exception if the architecure is not defined in this class. - * @return Arch - */ - public static Arch fromString(String value) throws IllegalStateException { - return fromValue(value); - } - - /** - * Compares if a given Arch object is equal to this. - * @param obj Object - * @return boolean - */ - public boolean equals(Object obj) { - return (obj == this); - } - - public int hashCode() { - return toString().hashCode(); - } - - /** - * Returns the string value of the architecture. - * @return String - */ - public String toString() { - return _value_; - } - } Modified: trunk/src/org/globus/swift/catalog/types/Os.java =================================================================== --- trunk/src/org/globus/swift/catalog/types/Os.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/globus/swift/catalog/types/Os.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -17,89 +17,6 @@ package org.globus.swift.catalog.types; -/** - * This is an enumerated data class for the different types of operating systems. - * - * @author Gaurang Mehta gmehta at isi.edu - * @version $Revision: 1.4 $ - */ - -import java.io.Serializable; -import java.util.HashMap; - -public class Os - implements Serializable { - private String _value_; - private static HashMap _table_ = new HashMap(5); - - protected Os(String value) { - _value_ = value; - _table_.put(_value_, this); - } - - private static final String _LINUX = "LINUX"; - private static final String _SUNOS = "SUNOS"; - private static final String _AIX = "AIX"; - private static final String _WINDOWS = "WINDOWS"; - - public static final Os LINUX = new Os(_LINUX); - public static final Os SUNOS = new Os(_SUNOS); - public static final Os AIX = new Os(_AIX); - public static final Os WINDOWS = new Os(_WINDOWS); - - public static final String err = "Error: Illegal Operating System defined. Please specify one of the predefined types \n [LINUX, SUNOS, AIX, WINDOWS]"; - - /** - * Returns the value of the operating system as string. - * @return String - */ - public String getValue() { - return _value_; - } - - /** - * Creates a new Os object given an os string. - * @param value String - * @throws IllegalStateException Throws Exception if the operating system is not defined in this class. - * @return Os - */ - public static Os fromValue(String value) throws IllegalStateException { - Os m_enum = _table_.get(value.toUpperCase()); - if (m_enum == null) { - throw new IllegalStateException(err); - } - return m_enum; - } - - /** - * Creates a new Os object given an os string. - * @param value String - * @throws IllegalStateException Throws Exception if the operating system is not defined in this class. - * @return Os - */ - public static Os fromString(String value) throws IllegalStateException { - return fromValue(value); - } - - /** - * Compares if a given Os object is equal to this. - * @param obj Object - * @return boolean - */ - public boolean equals(Object obj) { - return (obj == this); - } - - public int hashCode() { - return toString().hashCode(); - } - - /** - * Returns the string value of the operating system. - * @return String - */ - public String toString() { - return _value_; - } - +public enum Os { + LINUX, SUNOS, AIX, WINDOWS; } Modified: trunk/src/org/globus/swift/catalog/types/SysInfo.java =================================================================== --- trunk/src/org/globus/swift/catalog/types/SysInfo.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/globus/swift/catalog/types/SysInfo.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -77,8 +77,8 @@ * @param glibc String */ public SysInfo(String arch, String os, String osversion, String glibc) { - this.arch = (arch == null) ? Arch.INTEL32 : Arch.fromString(arch); - this.os = (os == null) ? Os.LINUX : Os.fromString(os); + this.arch = (arch == null) ? Arch.INTEL32 : Arch.valueOf(arch); + this.os = (os == null) ? Os.LINUX : Os.valueOf(os); this.osversion = (osversion == null || osversion.equals("") ) ? null: osversion; @@ -92,9 +92,9 @@ if (system != null) { String s1[] = system.split("::", 2); if (s1.length == 2) { - arch = Arch.fromString(s1[0]); + arch = Arch.valueOf(s1[0]); String s2[] = s1[1].split(":", 3); - os = Os.fromString(s2[0]); + os = Os.valueOf(s2[0]); for (int i = 1; i < s2.length; i++) { if (i == 1) { osversion = s2[i]; Modified: trunk/src/org/globus/swift/data/Director.java =================================================================== --- trunk/src/org/globus/swift/data/Director.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/globus/swift/data/Director.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -33,10 +33,10 @@ import java.util.regex.Pattern; import org.apache.log4j.Logger; -import org.globus.swift.data.policy.Policy; import org.globus.swift.data.policy.Broadcast; +import org.globus.swift.data.policy.Policy; import org.globus.swift.data.util.LineReader; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; /** * Manages CDM policies for files based on pattern matching. @@ -103,22 +103,20 @@ /** Read in the user-supplied CDM policy file. */ - public static void load(File file) throws IOException { + public static void load(File file, SwiftConfig config) throws IOException { logger.debug("CDM file: " + file); Director.policyFile = file; List list = LineReader.read(file); for (String s : list) addLine(s); - init(); + init(config); } - static void init() throws IOException - { - VDL2Config config = VDL2Config.getConfig(); + static void init(SwiftConfig config) throws IOException + { + broadcastMode = config.getStringProperty("cdm.broadcast.mode"); + logfile = config.getStringProperty("logfile"); - broadcastMode = config.getProperty("cdm.broadcast.mode"); - logfile = config.getProperty("logfile"); - if (broadcastMode.equals("file")) broadcasted.put("LOCAL_FILE", new HashSet()); @@ -313,7 +311,7 @@ logger.debug("Policy file does not exist: " + args[1]); } - load(policyFile); + load(policyFile, SwiftConfig.getDefault()); Policy policy = lookup(name); logger.debug(name + ": " + policy); } catch (Exception e) { Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -35,9 +35,11 @@ import java.security.SecureRandom; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.Enumeration; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -67,9 +69,8 @@ import org.griphyn.vdl.toolkit.VDLt2VDLx.IncorrectInvocationException; import org.griphyn.vdl.toolkit.VDLt2VDLx.ParsingException; import org.griphyn.vdl.util.LazyFileAppender; -import org.griphyn.vdl.util.VDL2Config; -import org.griphyn.vdl.util.VDL2ConfigProperties; -import org.griphyn.vdl.util.VDL2ConfigProperties.PropInfo; +import org.griphyn.vdl.util.SwiftConfig; +import org.griphyn.vdl.util.SwiftConfigSchema; public class Loader extends org.globus.cog.karajan.Loader { private static final Logger logger = Logger.getLogger(Loader.class); @@ -78,18 +79,37 @@ public static final String ARG_VERSION = "version"; public static final String ARG_RESUME = "resume"; public static final String ARG_INSTANCE_CONFIG = "config"; + public static final String ARG_SITELIST = "sitelist"; + public static final String ARG_SITES_FILE = "sites.file"; + public static final String ARG_TC_FILE = "tc.file"; public static final String ARG_DRYRUN = "dryrun"; public static final String ARG_VERBOSE = "verbose"; public static final String ARG_DEBUG = "debug"; public static final String ARG_LOGFILE = "logfile"; - public static final String ARG_CDMFILE = "cdm.file"; + public static final String ARG_CDMFILE = "cdmfile"; public static final String ARG_RUNID = "runid"; public static final String ARG_UI = "ui"; public static final String ARG_RECOMPILE = "recompile"; - public static final String ARG_REDUCED_LOGGING = "reduced.logging"; - public static final String ARG_MINIMAL_LOGGING = "minimal.logging"; - public static final String ARG_PAUSE_ON_START = "pause.on.start"; + public static final String ARG_REDUCED_LOGGING = "reducedLogging"; + public static final String ARG_MINIMAL_LOGGING = "minimalLogging"; + public static final String ARG_PAUSE_ON_START = "pauseOnStart"; public static final String ARG_EXECUTE = "e"; + + public static final List CMD_LINE_OPTIONS; + + static { + CMD_LINE_OPTIONS = new ArrayList(); + CMD_LINE_OPTIONS.add("hostName"); + CMD_LINE_OPTIONS.add("TCPPortRange"); + CMD_LINE_OPTIONS.add("lazyErrors"); + CMD_LINE_OPTIONS.add("keepSiteDir"); + CMD_LINE_OPTIONS.add("alwaysTransferWrapperLog"); + CMD_LINE_OPTIONS.add("logProvenance"); + CMD_LINE_OPTIONS.add("fileGCEnabled"); + CMD_LINE_OPTIONS.add("mappingCheckerEnabled"); + CMD_LINE_OPTIONS.add("tracingEnabled"); + CMD_LINE_OPTIONS.add("maxForeachThreads"); + } public static String buildVersion; @@ -120,18 +140,19 @@ source = null; } - VDL2Config config = loadConfig(ap); - addCommandLineProperties(config, ap); - config.validateProperties(); - setupLogging(ap, projectName, runID); + SwiftConfig config = loadConfig(ap, getCommandLineProperties(ap)); + if (ap.isPresent(ARG_SITELIST)) { + printSiteList(config); + System.exit(0); + } + setupLogging(ap, config, projectName, runID); logBasicInfo(argv, runID, config); - debugSitesText(config); - debugTCText(config); + debugConfigText(config); - boolean provenanceEnabled = VDL2Config.getConfig().getProvenanceLog(); + boolean provenanceEnabled = config.isProvenanceEnabled(); if (ap.isPresent(ARG_CDMFILE)) { - loadCDM(ap); + loadCDM(ap, config); } WrapperNode tree = null; @@ -235,7 +256,14 @@ System.exit(runerror ? 2 : 0); } - private static void logBasicInfo(String[] argv, String runID, VDL2Config conf) { + private static void printSiteList(SwiftConfig config) { + System.out.println("Available sites: "); + for (String name : config.getDefinedSiteNames()) { + System.out.println("\t" + name); + } + } + + private static void logBasicInfo(String[] argv, String runID, SwiftConfig conf) { String version = loadVersion(); System.out.println(version); System.out.println("RunID: " + runID); @@ -278,6 +306,18 @@ System.out.println(loadVersion()); System.exit(0); } + if (ap.isPresent(ARG_SITES_FILE)) { + System.err.println("Swift does not use site files any more.\n" + + "Please use the swift-convert-config tool to update your " + + "sites file to a swift configuration file."); + System.exit(1); + } + if (ap.isPresent(ARG_TC_FILE)) { + System.err.println("Swift does not use TC files any more.\n" + + "Please use the swift-convert-config tool to update your " + + "sites.xml and tc.data to a Swift configuration file."); + System.exit(1); + } if (!ap.hasValue(ArgumentParser.DEFAULT) && !ap.isPresent(ARG_EXECUTE)) { System.out.println(loadVersion()); error("No Swift script specified"); @@ -328,13 +368,13 @@ System.err.print("For usage information: swift -help\n\n"); } - static void loadCDM(ArgumentParser ap) { + static void loadCDM(ArgumentParser ap, SwiftConfig config) { String cdmString = null; try { cdmString = ap.getStringValue(ARG_CDMFILE); File cdmFile = new File(cdmString); debugText("CDM FILE", cdmFile); - Director.load(cdmFile); + Director.load(cdmFile, config); } catch (IOException e) { logger.debug("Detailed exception:", e); @@ -496,19 +536,10 @@ } } - static void debugSitesText(VDL2Config config) { - String poolFile = config.getPoolFile(); - logger.info("SITES_FILE " + poolFile); - debugText("SITES", new File(poolFile)); + static void debugConfigText(SwiftConfig config) { + logger.info("SWIFT_CONF \n" + config); } - static void debugTCText(VDL2Config config) { - String tcFile = config.getTCFile(); - logger.info("TC_FILE " + tcFile); - debugText("TC", new File(tcFile)); - } - - /** * The build ID is a UID that gets generated with each build. It is * used to decide whether an already compiled swift script can be @@ -530,29 +561,30 @@ } } - private static VDL2Config loadConfig(ArgumentParser ap) throws IOException { - VDL2Config conf; + private static SwiftConfig loadConfig(ArgumentParser ap, Map cmdLine) throws IOException { + SwiftConfig conf; if (ap.hasValue(ARG_INSTANCE_CONFIG)) { String configFile = ap.getStringValue(ARG_INSTANCE_CONFIG); - conf = VDL2Config.getConfig(configFile); + conf = SwiftConfig.load(configFile, cmdLine); + SwiftConfig.setDefault(conf); } else { - conf = (VDL2Config) VDL2Config.getConfig().clone(); + conf = (SwiftConfig) SwiftConfig.load().clone(); } return conf; } - private static void addCommandLineProperties(VDL2Config config, - ArgumentParser ap) { - config.setCurrentFile(""); - Map desc = VDL2ConfigProperties.getPropertyDescriptions(); - for (Map.Entry e : desc.entrySet()) { + private static Map getCommandLineProperties(ArgumentParser ap) { + Map cmdConf = new HashMap(); + Map desc = SwiftConfig.SCHEMA.getPropertyDescriptions(); + for (Map.Entry e : desc.entrySet()) { String name = e.getKey(); if (ap.isPresent(name)) { String value = ap.getStringValue(name); - config.setProperty(name, value); + cmdConf.put(name, value); } } + return cmdConf; } private static ArgumentParser buildArgumentParser() { @@ -574,6 +606,8 @@ ap.addFlag(ARG_DRYRUN, "Runs the SwiftScript program without submitting any jobs (can be used to get a graph)"); + ap.addHiddenFlag(ARG_SITES_FILE); + ap.addHiddenFlag(ARG_TC_FILE); ap.addOption(ARG_RESUME, "Resumes the execution using a log file", "file", ArgumentParser.OPTIONAL); @@ -583,6 +617,7 @@ "If individual command line arguments are used for properties, they will override " + "the contents of this file.", "file", ArgumentParser.OPTIONAL); + ap.addFlag(ARG_SITELIST, "Prints a list of sites available in the swift configuration"); ap.addFlag(ARG_VERBOSE, "Increases the level of output that Swift produces on the console to include more detail " + "about the execution"); @@ -621,10 +656,10 @@ ap.addOption(ARG_EXECUTE, "Runs the swift script code contained in ", "string", ArgumentParser.OPTIONAL); - Map desc = VDL2ConfigProperties.getPropertyDescriptions(); - for (Map.Entry e : desc.entrySet()) { - PropInfo pi = e.getValue(); - ap.addOption(e.getKey(), pi.desc, pi.validValues, + Map desc = SwiftConfig.SCHEMA.getPropertyDescriptions(); + for (Map.Entry e : desc.entrySet()) { + SwiftConfigSchema.Info pi = e.getValue(); + ap.addOption(e.getKey(), pi.doc, pi.type.toString(), ArgumentParser.OPTIONAL); } return ap; @@ -632,7 +667,7 @@ private static MonitorAppender ma; - protected static void setupLogging(ArgumentParser ap, String projectName, + protected static String setupLogging(ArgumentParser ap, SwiftConfig config, String projectName, String runID) throws IOException { String logfile; if (ap.isPresent(ARG_LOGFILE)) { @@ -641,10 +676,9 @@ else { logfile = projectName + "-" + runID + ".log"; } - - VDL2Config config = VDL2Config.getConfig(); - config.put("logfile", logfile); + config.setProperty("logfile", logfile); + File f = new File(logfile); FileAppender fa = (FileAppender) getAppender(FileAppender.class); @@ -698,6 +732,7 @@ Logger.getLogger(New.class).setLevel(Level.WARN); Logger.getLogger("org.globus.cog.karajan.workflow.service").setLevel(Level.WARN); } + return logfile; } Modified: trunk/src/org/griphyn/vdl/karajan/SwiftRootScope.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/SwiftRootScope.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/SwiftRootScope.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -17,14 +17,14 @@ import org.globus.cog.karajan.util.KarajanProperties; import org.griphyn.vdl.karajan.lib.swiftscript.FnArg; import org.griphyn.vdl.mapping.DuplicateMappingChecker; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class SwiftRootScope extends RootScope { public SwiftRootScope(KarajanProperties props, String file, Context context) { super(props, file, context); context.setAttribute("SWIFT:DM_CHECKER", new DuplicateMappingChecker( - (VDL2Config) context.getAttribute("SWIFT:CONFIG"))); + (SwiftConfig) context.getAttribute("SWIFT:CONFIG"))); addVar("PATH_SEPARATOR", File.separator); addVar("SWIFT:DRY_RUN", context.getAttribute("SWIFT:DRY_RUN")); Modified: trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -52,6 +52,7 @@ import org.globus.cog.karajan.util.ContactSet; import org.globus.cog.karajan.util.TypeUtil; import org.globus.swift.catalog.site.SwiftContact; +import org.griphyn.vdl.util.SwiftConfig; public class VDSAdaptiveScheduler extends WeightedHostScoreScheduler implements CoasterResourceTracker { @@ -78,7 +79,7 @@ serviceContactMapping = new HashMap(); } - public static final String PROP_TC_FILE = "transformationCatalogFile"; + public static final String PROP_CONFIG = "config"; public static final String PROP_CLUSTERING_ENABLED = "clusteringEnabled"; public static final String PROP_CLUSTERING_QUEUE_DELAY = "clusteringQueueDelay"; public static final String PROP_CLUSTERING_MIN_TIME = "clusteringMinTime"; @@ -88,15 +89,15 @@ public synchronized String[] getPropertyNames() { if (propertyNames == null) { propertyNames = AbstractScheduler.combineNames(super.getPropertyNames(), - new String[] { PROP_TC_FILE }); + new String[] { PROP_CONFIG }); } return propertyNames; } public void setProperty(String name, Object value) { - if (PROP_TC_FILE.equals(name)) { + if (PROP_CONFIG.equals(name)) { this.setConstraintChecker(new SwiftSiteChecker()); - this.addTaskTransformer(new VDSTaskTransformer()); + this.addTaskTransformer(new VDSTaskTransformer((SwiftConfig) value)); } else if (PROP_CLUSTERING_QUEUE_DELAY.equals(name)) { clusteringQueueDelay = TypeUtil.toInt(value); Modified: trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -20,7 +20,6 @@ */ package org.griphyn.vdl.karajan; -import java.io.IOException; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -34,15 +33,16 @@ import org.globus.cog.karajan.scheduler.TaskTransformer; import org.globus.cog.karajan.util.BoundContact; import org.globus.cog.karajan.util.Contact; -import org.griphyn.vdl.util.VDL2Config; +import org.globus.swift.catalog.site.SwiftContact; +import org.griphyn.vdl.util.SwiftConfig; public class VDSTaskTransformer implements TaskTransformer { public static final Logger logger = Logger.getLogger(VDSTaskTransformer.class); private TaskTransformer impl; - public VDSTaskTransformer() { - this.impl = new SwiftTransformer(); + public VDSTaskTransformer(SwiftConfig config) { + this.impl = new SwiftTransformer(config); } public void transformTask(Task task, Contact[] contacts, Service[] services) { @@ -50,8 +50,13 @@ } public static abstract class AbstractTransformer implements TaskTransformer { + private SwiftConfig config; - public void transformTask(Task task, Contact[] contacts, Service[] services) { + public AbstractTransformer(SwiftConfig config) { + this.config = config; + } + + public void transformTask(Task task, Contact[] contacts, Service[] services) { if (task.getType() == Task.JOB_SUBMISSION) { applyJobWorkDirectory(task, contacts); } @@ -117,7 +122,7 @@ private void applyJobWorkDirectory(Task task, Contact[] contacts) { JobSpecification spec = (JobSpecification) task.getSpecification(); String dir = spec.getDirectory(); - BoundContact bc = (BoundContact) contacts[0]; + SwiftContact bc = (SwiftContact) contacts[0]; String workdir = (String) bc.getProperty("workdir"); if (workdir==null){ @@ -141,24 +146,24 @@ // undergo this substitution... String executable = l.get(0); - try { - VDL2Config config = VDL2Config.getConfig(); + String mode = (String) bc.getProperty(SwiftConfig.Key.WRAPPER_INVOCATION_MODE.propName); + if (mode == null) { + mode = config.getWrapperInvocationMode(); + } + if (mode.equals("absolute") + && (executable.endsWith("shared/_swiftwrap") + || executable.endsWith("shared/_swiftseq"))) { - if (config.getProperty("wrapper.invocation.mode", bc).equals("absolute") - && (executable.endsWith("shared/_swiftwrap") - || executable.endsWith("shared/_swiftseq"))) { - - String s = spec.getDirectory() + "/" + executable; - l.set(0, s); - } - } - catch(IOException ioe) { - throw new RuntimeException("Could not determine wrapper invocation mode", ioe); + String s = spec.getDirectory() + "/" + executable; + l.set(0, s); } } } public static class SwiftTransformer extends AbstractTransformer { + public SwiftTransformer(SwiftConfig config) { + super(config); + } } } Modified: trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -20,8 +20,6 @@ */ package org.griphyn.vdl.karajan.functions; -import java.io.IOException; - import k.rt.Context; import k.rt.ExecutionException; import k.rt.Stack; @@ -39,7 +37,7 @@ import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.parser.WrapperNode; import org.globus.cog.karajan.util.BoundContact; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class ConfigProperty extends InternalFunction { private ArgRef name; @@ -47,7 +45,7 @@ private ArgRef host; private ChannelRef cr_vargs; - private VDL2Config instanceConfig; + private SwiftConfig instanceConfig; private VarRef context; @@ -68,9 +66,12 @@ context = scope.getVarRef("#context"); Context ctx = context.getValue(); Var.Channel r = scope.parent.lookupChannel("..."); + if (this.name.isStatic()) { + checkProperty(this.name.getValue()); + } if (this.name.isStatic() && this.instance.isStatic() && this.host.isStatic() && ctx != null && this.host.getValue() == null) { - String value = getProperty(this.name.getValue(), this.instance.getValue(), getInstanceConfig(ctx)); + Object value = getProperty(this.name.getValue(), this.instance.getValue(), getInstanceConfig(ctx)); if (r.append(value)) { return null; } @@ -108,44 +109,45 @@ cr_vargs.append(stack, getProperty(name, instance, getInstanceConfig(stack))); } - private synchronized VDL2Config getInstanceConfig(Stack stack) { + private synchronized SwiftConfig getInstanceConfig(Stack stack) { if (instanceConfig == null) { Context ctx = this.context.getValue(stack); - instanceConfig = (VDL2Config) ctx.getAttribute("SWIFT:CONFIG"); + instanceConfig = (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG"); } return instanceConfig; } - private synchronized VDL2Config getInstanceConfig(Context ctx) { - return (VDL2Config) ctx.getAttribute("SWIFT:CONFIG"); + private synchronized SwiftConfig getInstanceConfig(Context ctx) { + return (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG"); } - public static String getProperty(String name, VDL2Config instanceConfig) { + public static Object getProperty(String name, SwiftConfig instanceConfig) { return getProperty(name, true, instanceConfig); } - public static String getProperty(String name, boolean instance, VDL2Config instanceConfig) { - try { - VDL2Config conf; - String prop; - if (!instance) { - conf = VDL2Config.getConfig(); - prop = conf.getProperty(name); - } - else { - conf = instanceConfig; - prop = conf.getProperty(name); - } - if (prop == null) { - throw new ExecutionException("Swift config property \"" + name + "\" not found in " - + conf); - } - else { - return prop; - } + public static Object getProperty(String name, boolean instance, SwiftConfig instanceConfig) { + SwiftConfig conf; + Object prop; + if (!instance) { + conf = SwiftConfig.getDefault(); + prop = conf.getProperty(name); } - catch (IOException e) { - throw new ExecutionException("Failed to load Swift configuration", e); + else { + conf = instanceConfig; + prop = conf.getProperty(name); } + if (prop == null) { + throw new ExecutionException("Swift config property \"" + name + "\" not found in " + + conf.getFileName()); + } + else { + return prop; + } } + + private void checkProperty(String name) throws CompilationException { + if (!SwiftConfig.SCHEMA.propertyExists(name)) { + throw new CompilationException(this, "Unknown configuration property: " + name); + } + } } Modified: trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -76,7 +76,10 @@ VDL2ErrorTranslator translator = VDL2ErrorTranslator.getDefault(); Map count = new HashMap(); - for (ExecutionException ex : l) { + for (ExecutionException ex : l) { + if (ex == null) { + continue; + } if (ex.getCause() instanceof ConcurrentModificationException) { ex.printStackTrace(); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/CacheFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/CacheFunction.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/CacheFunction.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -26,14 +26,12 @@ import org.globus.cog.karajan.analyzer.CompilationException; import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.VarRef; -import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.compiled.nodes.InternalFunction; +import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.parser.WrapperNode; -import org.griphyn.vdl.karajan.functions.ConfigProperty; import org.griphyn.vdl.karajan.lib.cache.VDLFileCache; import org.griphyn.vdl.karajan.lib.cache.VDLFileCacheFactory; -import org.griphyn.vdl.util.VDL2Config; -import org.griphyn.vdl.util.VDL2ConfigProperties; +import org.griphyn.vdl.util.SwiftConfig; public abstract class CacheFunction extends InternalFunction { public static final String CACHE_FILES_TO_REMOVE = "cacheFilesToRemove"; @@ -62,8 +60,8 @@ synchronized(ctx) { VDLFileCache cache = (VDLFileCache) ctx.getAttribute("SWIFT:FILE_CACHE"); if (cache == null) { - cache = VDLFileCacheFactory.newInstance(ConfigProperty.getProperty( - VDL2ConfigProperties.CACHING_ALGORITHM, (VDL2Config) ctx.getAttribute("SWIFT:CONFIG"))); + SwiftConfig conf = (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG"); + cache = VDLFileCacheFactory.newInstance(conf.getCachingAlgorithm()); ctx.setAttribute("SWIFT:FILE_CACHE", cache); } return cache; Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -52,7 +52,7 @@ import org.griphyn.vdl.karajan.lib.RuntimeStats.ProgressState; import org.griphyn.vdl.karajan.lib.replication.CanceledReplicaException; import org.griphyn.vdl.karajan.lib.replication.ReplicationManager; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class Execute extends GridExec { public static final Logger logger = Logger.getLogger(Execute.class); @@ -66,6 +66,7 @@ private VarRef context; private boolean replicationEnabled; + private SwiftConfig config; @Override protected Signature getSignature() { @@ -103,8 +104,8 @@ protected void addLocals(Scope scope) { super.addLocals(scope); context = scope.getVarRef("#context"); - VDL2Config config = (VDL2Config) context.getValue().getAttribute("SWIFT:CONFIG"); - replicationEnabled = "true".equals(config.getProperty("replication.enabled")); + config = (SwiftConfig) context.getValue().getAttribute("SWIFT:CONFIG"); + replicationEnabled = config.isReplicationEnabled(); } @Override @@ -232,7 +233,7 @@ synchronized (ctx) { ReplicationManager rm = (ReplicationManager) ctx.getAttribute("#replicationManager"); if (rm == null) { - rm = new ReplicationManager(getScheduler(stack)); + rm = new ReplicationManager(getScheduler(stack), config); ctx.setAttribute("#replicationManager", rm); } return rm; Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -27,8 +27,7 @@ import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.VarRef; import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; -import org.griphyn.vdl.karajan.functions.ConfigProperty; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class GetURLPrefix extends AbstractSingleValuedFunction { private VarRef context; @@ -49,8 +48,8 @@ @Override public Object function(Stack stack) { Context ctx = this.context.getValue(stack); - String localServerBase = ConfigProperty.getProperty("wrapper.staging.local.server", - (VDL2Config) ctx.getAttribute("SWIFT:CONFIG")); + SwiftConfig config = (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG"); + String localServerBase = config.getWrapperStagingLocalServer(); String cwd = this.cwd.getValue(stack); if (cwd.endsWith("/.")) { Modified: trunk/src/org/griphyn/vdl/karajan/lib/New.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/New.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/New.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -59,9 +59,7 @@ public static final Logger logger = Logger.getLogger(New.class); private static final Mapper NULL_MAPPER = new NullMapper(); - - private static DuplicateMappingChecker staticDMC = new DuplicateMappingChecker(null); - + private ArgRef field; private ArgRef mapping; private ArgRef value; Modified: trunk/src/org/griphyn/vdl/karajan/lib/Operators.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Operators.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/Operators.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -28,22 +28,11 @@ import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class Operators { - public static final boolean PROVENANCE_ENABLED; - - static { - boolean v; - try { - v = VDL2Config.getConfig().getProvenanceLog(); - } - catch (Exception e) { - v = false; - } - PROVENANCE_ENABLED = v; - } + public static final boolean PROVENANCE_ENABLED = SwiftConfig.getDefault().isProvenanceEnabled(); public static final Logger provenanceLogger = Logger.getLogger("org.globus.swift.provenance.operators"); Modified: trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -20,6 +20,7 @@ */ package org.griphyn.vdl.karajan.lib; +import k.rt.Context; import k.rt.Stack; import k.thr.LWThread; @@ -29,11 +30,10 @@ import org.globus.cog.karajan.analyzer.Scope; import org.globus.cog.karajan.analyzer.Signature; import org.globus.cog.karajan.analyzer.VarRef; +import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.globus.cog.karajan.compiled.nodes.Node; -import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.globus.cog.karajan.parser.WrapperNode; -import org.griphyn.vdl.karajan.functions.ConfigProperty; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class Parameterlog extends InternalFunction { public static final Logger logger = Logger.getLogger(Parameterlog.class); @@ -48,12 +48,12 @@ } private Boolean enabled; - private VarRef config; + private VarRef context; @Override protected Node compileBody(WrapperNode w, Scope argScope, Scope scope) throws CompilationException { - config = scope.getVarRef("SWIFT_CONFIG"); + context = scope.getVarRef("#context"); return super.compileBody(w, argScope, scope); } @@ -65,7 +65,9 @@ boolean run; synchronized(this) { if (enabled == null) { - enabled = "true".equals(ConfigProperty.getProperty("provenance.log", true, config.getValue(stack))); + Context ctx = this.context.getValue(stack); + SwiftConfig config = (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG"); + enabled = config.isProvenanceEnabled(); } run = enabled; } Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -17,7 +17,6 @@ package org.griphyn.vdl.karajan.lib; -import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; @@ -27,7 +26,6 @@ import java.util.Set; import k.rt.Context; -import k.rt.ExecutionException; import k.rt.Stack; import k.thr.LWThread; @@ -42,26 +40,15 @@ import org.globus.cog.karajan.compiled.nodes.InternalFunction; import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.parser.WrapperNode; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; /** this is an icky class that does too much with globals, but is for proof of concept. */ public class RuntimeStats { - public static final boolean TICKER_DISABLED; + public static final boolean TICKER_DISABLED = !SwiftConfig.getDefault().isTickerEnabled(); - static{ - boolean disabled; - try{ - disabled = "true".equalsIgnoreCase(VDL2Config.getConfig().getProperty("ticker.disable")); - } - catch (Exception e) { - disabled = false; - } - TICKER_DISABLED = disabled; - } - public static final String TICKER = "SWIFT_TICKER"; //formatter for timestamp against std.err lines @@ -105,13 +92,7 @@ t.start(); context.getValue(thr.getStack()).setAttribute(TICKER, t); // Allow user to reformat output date - String format; - try { - format = VDL2Config.getDefaultConfig().getTickerDateFormat(); - } - catch (IOException e) { - throw new ExecutionException(this, e); - } + String format = SwiftConfig.getDefault().getTickerDateFormat(); if (format != null && format.length() > 0) { formatter = new SimpleDateFormat(format); } @@ -253,17 +234,11 @@ public ProgressTicker() { super("Progress ticker"); states = new HashSet(); - try { - if ("true".equalsIgnoreCase(VDL2Config.getConfig().getProperty("ticker.disable"))) { - logger.info("Ticker disabled in configuration file"); - disabled = true; - } - tickerPrefix = - VDL2Config.getConfig().getTickerPrefix(); + if (!SwiftConfig.getDefault().isTickerEnabled()) { + logger.info("Ticker disabled in configuration file"); + disabled = true; } - catch (IOException e) { - logger.debug("Could not read swift properties", e); - } + tickerPrefix =SwiftConfig.getDefault().getTickerPrefix(); start = System.currentTimeMillis(); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -9,412 +9,24 @@ */ package org.griphyn.vdl.karajan.lib; -import java.util.ArrayList; -import java.util.List; - -import k.rt.ExecutionException; import k.rt.Stack; -import org.globus.cog.abstraction.impl.common.AbstractionFactory; -import org.globus.cog.abstraction.impl.common.ProviderMethodException; -import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl; -import org.globus.cog.abstraction.impl.common.task.InvalidProviderException; -import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl; -import org.globus.cog.abstraction.impl.common.task.ServiceImpl; -import org.globus.cog.abstraction.interfaces.ExecutionService; -import org.globus.cog.abstraction.interfaces.Service; -import org.globus.cog.abstraction.interfaces.ServiceContact; import org.globus.cog.karajan.analyzer.ArgRef; import org.globus.cog.karajan.analyzer.Param; import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction; -import org.globus.swift.catalog.site.Application; -import org.globus.swift.catalog.site.SiteCatalogParser; -import org.globus.swift.catalog.site.SwiftContact; -import org.globus.swift.catalog.site.SwiftContactSet; -import org.w3c.dom.Document; -import org.w3c.dom.NamedNodeMap; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; +import org.griphyn.vdl.util.SwiftConfig; public class SiteCatalog extends AbstractSingleValuedFunction { - private ArgRef fileName; + private ArgRef config; @Override protected Param[] getParams() { - return params("fileName"); + return params("config"); } @Override public Object function(Stack stack) { - String fn = fileName.getValue(stack); - SiteCatalogParser p = new SiteCatalogParser(fn); - try { - Document doc = p.parse(); - return buildResources(doc); - } - catch (Exception e) { - throw new ExecutionException(this, "Failed to parse site catalog", e); - } + SwiftConfig config = this.config.getValue(stack); + return config.getSites(); } - - private static class KVPair { - public final String key; - public final String value; - - public KVPair(String key, String value) { - this.key = key; - this.value = value; - } - } - - private Object buildResources(Document doc) { - Node root = getRoot(doc); - - if (root.getLocalName().equals("config")) { - throw new IllegalArgumentException("Old sites file format. Please upgrade your sites file to the new format."); - } - else if (root.getLocalName().equals("sites")) { - return parse(root); - } - else { - throw new IllegalArgumentException("Illegal sites file root node: " + root.getLocalName()); - } - } - - private Object parse(Node config) { - SwiftContactSet cs = new SwiftContactSet(); - NodeList pools = config.getChildNodes(); - for (int i = 0; i < pools.getLength(); i++) { - Node n = pools.item(i); - if (n.getNodeType() == Node.ELEMENT_NODE) { - String ctype = n.getNodeName(); - if (ctype.equals("site")) { - try { - SwiftContact bc = pool(n); - if (bc != null) { - cs.addContact(bc); - } - } - catch (Exception e) { - throw new ExecutionException(this, "Invalid site entry '" + poolName(n) + "': ", e); - } - } - else if (ctype.equals("apps")) { - SwiftContact dummy = new SwiftContact(); - apps(dummy, n); - for (Application app : dummy.getApplications()) { - cs.addApplication(app); - } - } - else { - throw new IllegalArgumentException("Invalid node: " + ctype); - } - } - } - return cs; - } - - private String poolName(Node site) { - if (site.getLocalName().equals("site")) { - return attr(site, "name"); - } - else { - throw new IllegalArgumentException("Invalid node: " + site.getLocalName()); - } - } - - private Node getRoot(Document doc) { - NodeList l = doc.getChildNodes(); - for (int i = 0; i < l.getLength(); i++) { - if (l.item(i).getNodeType() == Node.ELEMENT_NODE) { - return l.item(i); - } - } - throw new IllegalArgumentException("Missing root element"); - } - - private SwiftContact pool(Node n) throws InvalidProviderException, ProviderMethodException { - if (n.getNodeType() != Node.ELEMENT_NODE) { - return null; - } - String name = poolName(n); - SwiftContact bc = new SwiftContact(name); - - String sysinfo = attr(n, "sysinfo", null); - if (sysinfo != null) { - bc.setProperty("sysinfo", sysinfo); - } - - NodeList cs = n.getChildNodes(); - - for (int i = 0; i < cs.getLength(); i++) { - Node c = cs.item(i); - if (c.getNodeType() != Node.ELEMENT_NODE) { - continue; - } - String ctype = c.getNodeName(); - - if (ctype.equals("execution")) { - bc.addService(execution(c)); - } - else if (ctype.equals("filesystem")) { - bc.addService(filesystem(c)); - } - else if (ctype.equals("workdirectory")) { - bc.setProperty("workdir", text(c)); - } - else if (ctype.equals("scratch")) { - bc.setProperty("scratch", text(c)); - } - else if (ctype.equals("property")) { - bc.setProperty(attr(c, "name"), text(c)); - } - else if (ctype.equals("apps")) { - apps(bc, c); - } - else { - throw new IllegalArgumentException("Unknown node type: " + ctype); - } - } - return bc; - } - - private void apps(SwiftContact bc, Node n) { - NodeList cs = n.getChildNodes(); - - List envs = new ArrayList(); - List props = new ArrayList(); - for (int i = 0; i < cs.getLength(); i++) { - Node c = cs.item(i); - if (c.getNodeType() != Node.ELEMENT_NODE) { - continue; - } - String ctype = c.getNodeName(); - - if (ctype.equals("app")) { - bc.addApplication(application(c)); - } - else if (ctype.equals("env")) { - envs.add(env(c)); - } - else if (ctype.equals("property")) { - props.add(this.property(c)); - } - else { - throw new IllegalArgumentException("Unknown node type: " + ctype); - } - } - - mergeEnvsToApps(bc, envs); - mergePropsToApps(bc, props); - } - - private void mergeEnvsToApps(SwiftContact bc, List envs) { - for (Application app : bc.getApplications()) { - for (KVPair kvp : envs) { - if (!app.getEnv().containsKey(kvp.key)) { - // only merge if app does not override - app.setEnv(kvp.key, kvp.value); - } - } - } - } - - private void mergePropsToApps(SwiftContact bc, List props) { - for (Application app : bc.getApplications()) { - for (KVPair kvp : props) { - if (!app.getProperties().containsKey(kvp.key)) { - app.addProperty(kvp.key, kvp.value); - } - } - } - } - - private Application application(Node n) { - Application app = new Application(); - app.setName(attr(n, "name")); - app.setExecutable(attr(n, "executable")); - - NodeList cs = n.getChildNodes(); - - for (int i = 0; i < cs.getLength(); i++) { - Node c = cs.item(i); - if (c.getNodeType() != Node.ELEMENT_NODE) { - continue; - } - String ctype = c.getNodeName(); - - if (ctype.equals("env")) { - KVPair env = env(c); - app.setEnv(env.key, env.value); - } - else if (ctype.equals("property")) { - KVPair prop = property(c); - app.addProperty(prop.key, prop.value); - } - else { - throw new IllegalArgumentException("Unknown node type: " + ctype); - } - } - - return app; - } - - private Service execution(Node n) throws InvalidProviderException, ProviderMethodException { - String provider = attr(n, "provider"); - String url = attr(n, "url", null); - String jobManager = attr(n, "jobManager", null); - if (jobManager == null) { - jobManager = attr(n, "jobmanager", null); - } - - ExecutionService s = new ExecutionServiceImpl(); - s.setProvider(provider); - ServiceContact contact = null; - if (url != null) { - contact = new ServiceContactImpl(url); - s.setServiceContact(contact); - s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact)); - } - - if (jobManager != null) { - s.setJobManager(jobManager); - } - - properties(s, n); - - return s; - } - - private void properties(Service s, Node n) { - NodeList cs = n.getChildNodes(); - - for (int i = 0; i < cs.getLength(); i++) { - Node c = cs.item(i); - if (c.getNodeType() != Node.ELEMENT_NODE) { - continue; - } - String ctype = c.getNodeName(); - - if (ctype.equals("property")) { - property(s, c); - } - else { - throw new IllegalArgumentException("Unknown node type: " + ctype); - } - } - - } - - private void property(Service s, Node c) { - s.setAttribute(attr(c, "name"), text(c)); - } - - private KVPair property(Node c) { - return new KVPair(attr(c, "name"), text(c)); - } - - private Service filesystem(Node n) throws InvalidProviderException, ProviderMethodException { - String provider = attr(n, "provider"); - String url = attr(n, "url", null); - - Service s = new ServiceImpl(); - s.setType(Service.FILE_OPERATION); - s.setProvider(provider); - - ServiceContact contact = null; - if (url != null) { - contact = new ServiceContactImpl(url); - s.setServiceContact(contact); - s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact)); - } - - properties(s, n); - - return s; - } - - private KVPair env(Node n) { - String key = attr(n, "name"); - String value = text(n); - - return new KVPair(key, value); - } - - private String text(Node n) { - if (n.getFirstChild() != null) { - return n.getFirstChild().getNodeValue(); - } - else { - return null; - } - } - - private String attr(Node n, String name) { - NamedNodeMap attrs = n.getAttributes(); - if (attrs != null) { - Node attr = attrs.getNamedItem(name); - if (attr == null) { - throw new IllegalArgumentException("Missing " + name); - } - else { - return expandProps(attr.getNodeValue()); - } - } - else { - throw new IllegalArgumentException("Missing " + name); - } - } - - private String attr(Node n, String name, String defVal) { - NamedNodeMap attrs = n.getAttributes(); - if (attrs != null) { - Node attr = attrs.getNamedItem(name); - if (attr == null) { - return defVal; - } - else { - return expandProps(attr.getNodeValue()); - } - } - else { - return defVal; - } - } - - private String expandProps(String v) { - if (v == null) { - return null; - } - StringBuilder sb = new StringBuilder(); - int li = -1; - for (int i = 0; i < v.length(); i++) { - char c = v.charAt(i); - switch (c) { - case '{': - if (li != -1) { - li = -1; - sb.append('{'); - } - else { - li = i; - } - break; - case '}': - if (li != -1) { - sb.append(System.getProperty(v.substring(li + 1, i))); - li = -1; - } - else { - sb.append(c); - } - break; - default: - if (li == -1) { - sb.append(c); - } - } - } - return sb.toString(); - } } Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -54,6 +54,8 @@ return getSingle(bc, name.getValue(stack), _default.getValue(stack)); } + public static final Os DEFAULT_OS = Os.LINUX; + public static final String SWIFT_WRAPPER_INTERPRETER = "wrapperInterpreter"; public static final String SWIFT_WRAPPER_INTERPRETER_OPTIONS = "wrapperInterpreterOptions"; public static final String SWIFT_WRAPPER_SCRIPT = "wrapperScript"; @@ -141,9 +143,9 @@ } private Os getOS(SwiftContact bc) { - Object o = bc.getProperty("sysinfo"); + Object o = bc.getProperty("OS"); if (o == null) { - return Os.LINUX; + return DEFAULT_OS; } else { return SysInfo.fromString(o.toString()).getOs(); Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -17,7 +17,6 @@ package org.griphyn.vdl.karajan.lib; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -56,25 +55,13 @@ import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; import org.griphyn.vdl.type.Types; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public abstract class SwiftFunction extends AbstractFunction { public static final Logger logger = Logger.getLogger(SwiftFunction.class); - public static final boolean PROVENANCE_ENABLED; - - static { - boolean v; - try { - v = VDL2Config.getConfig().getProvenanceLog(); - } - catch (IOException e) { - v = false; - } - PROVENANCE_ENABLED = v; - } + public static final boolean PROVENANCE_ENABLED = SwiftConfig.getDefault().isProvenanceEnabled(); - private VarRef context; @Override Modified: trunk/src/org/griphyn/vdl/karajan/lib/Throttled.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Throttled.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/Throttled.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -20,7 +20,6 @@ */ package org.griphyn.vdl.karajan.lib; -import java.io.IOException; import java.util.LinkedList; import k.rt.ConditionalYield; @@ -29,8 +28,7 @@ import k.thr.Yield; import org.globus.cog.karajan.compiled.nodes.Sequential; -import org.globus.cog.karajan.util.TypeUtil; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class Throttled extends Sequential { public static final int DEFAULT_MAX_THREADS = 1000000; @@ -39,13 +37,7 @@ private int maxThreadCount, current; public Throttled() { - try { - maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig() - .getProperty("max.threads", String.valueOf(DEFAULT_MAX_THREADS))); - } - catch (IOException e) { - maxThreadCount = DEFAULT_MAX_THREADS; - } + maxThreadCount = (Integer) SwiftConfig.getDefault().getProperty("maxThreads", DEFAULT_MAX_THREADS); current = 0; waiting = new LinkedList(); } Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -17,7 +17,6 @@ package org.griphyn.vdl.karajan.lib; -import java.io.IOException; import java.util.Iterator; import java.util.List; @@ -41,8 +40,7 @@ import org.globus.cog.karajan.compiled.nodes.Node; import org.globus.cog.karajan.compiled.nodes.UParallelFor; import org.globus.cog.karajan.parser.WrapperNode; -import org.globus.cog.karajan.util.TypeUtil; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class ThrottledParallelFor extends UParallelFor { public static final Logger logger = Logger @@ -229,13 +227,7 @@ private int getMaxThreads() { if (maxThreadCount < 0) { - try { - maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig() - .getProperty("foreach.max.threads", String.valueOf(DEFAULT_MAX_THREADS))); - } - catch (IOException e) { - maxThreadCount = DEFAULT_MAX_THREADS; - } + maxThreadCount = SwiftConfig.getDefault().getForeachMaxThreads(); } return maxThreadCount; } Modified: trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -9,7 +9,6 @@ */ package org.griphyn.vdl.karajan.lib; -import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -25,7 +24,7 @@ import org.griphyn.vdl.mapping.RootHandle; import org.griphyn.vdl.mapping.nodes.AbstractDataNode; import org.griphyn.vdl.type.Types; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class Tracer { public static final Logger logger = Logger.getLogger("TRACE"); @@ -34,12 +33,7 @@ private ThreadLocal thread = new ThreadLocal(); static { - try { - globalTracingEnabled = VDL2Config.getConfig().isTracingEnabled(); - } - catch (IOException e) { - globalTracingEnabled = false; - } + globalTracingEnabled = SwiftConfig.getDefault().isTracingEnabled(); NAME_MAPPINGS = new HashMap(); NAME_MAPPINGS.put("assignment", "ASSIGN"); NAME_MAPPINGS.put("iterate", "ITERATE"); Modified: trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -29,7 +29,7 @@ import org.globus.cog.abstraction.interfaces.JobSpecification; import org.globus.cog.abstraction.interfaces.Task; import org.globus.cog.karajan.scheduler.Scheduler; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class ReplicationManager { public static final Logger logger = Logger @@ -52,19 +52,15 @@ private ReplicationGroups replicationGroups; private Scheduler scheduler; - public ReplicationManager(Scheduler scheduler) { + public ReplicationManager(Scheduler scheduler, SwiftConfig config) { this.replicationGroups = new ReplicationGroups(scheduler); this.scheduler = scheduler; queued = new HashMap(); running = new HashMap(); try { - minQueueTime = Integer.parseInt(VDL2Config.getConfig().getProperty( - "replication.min.queue.time")); - enabled = Boolean.valueOf( - VDL2Config.getConfig().getProperty("replication.enabled")) - .booleanValue(); - limit = Integer.parseInt(VDL2Config.getConfig().getProperty( - "replication.limit")); + minQueueTime = config.getReplicationMinQueueTime(); + enabled = config.isReplicationEnabled(); + limit = config.getReplicationLimit(); } catch (Exception e) { logger.warn( Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java =================================================================== --- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -18,7 +18,6 @@ package org.griphyn.vdl.karajan.lib.swiftscript; import java.io.BufferedReader; -import java.io.IOException; import java.io.InputStreamReader; import java.util.Arrays; import java.util.Map; @@ -45,24 +44,13 @@ import org.griphyn.vdl.mapping.nodes.NodeFactory; import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Types; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; public class Misc { private static final Logger logger = Logger.getLogger(Misc.class); - public static final boolean PROVENANCE_ENABLED; - - static { - boolean v; - try { - v = VDL2Config.getConfig().getProvenanceLog(); - } - catch (IOException e) { - v = false; - } - PROVENANCE_ENABLED = v; - } + public static final boolean PROVENANCE_ENABLED = SwiftConfig.getDefault().isProvenanceEnabled(); private static final Logger traceLogger = Logger.getLogger("org.globus.swift.trace"); Modified: trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -16,8 +16,7 @@ import org.apache.log4j.Logger; import org.griphyn.vdl.mapping.nodes.AbstractDataNode; -import org.griphyn.vdl.util.VDL2Config; -import org.griphyn.vdl.util.VDL2ConfigProperties; +import org.griphyn.vdl.util.SwiftConfig; public class DuplicateMappingChecker { public static final Logger logger = Logger.getLogger(DuplicateMappingChecker.class); @@ -26,13 +25,8 @@ private final Map map; - public DuplicateMappingChecker(VDL2Config conf) { - if (conf == null) { - enabled = true; - } - else { - enabled = !"off".equals(conf.getProperty(VDL2ConfigProperties.DM_CHECKER)); - } + public DuplicateMappingChecker(SwiftConfig conf) { + enabled = conf.isMappingCheckerEnabled(); map = new HashMap(); } Modified: trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -17,7 +17,6 @@ package org.griphyn.vdl.mapping.file; -import java.io.IOException; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedList; @@ -27,8 +26,7 @@ import org.apache.log4j.Logger; import org.griphyn.vdl.mapping.PhysicalFormat; -import org.griphyn.vdl.util.VDL2Config; -import org.griphyn.vdl.util.VDL2ConfigProperties; +import org.griphyn.vdl.util.SwiftConfig; public class FileGarbageCollector implements Runnable { public static final Logger logger = Logger.getLogger(FileGarbageCollector.class); @@ -52,13 +50,7 @@ queue = new LinkedList(); usageCount = new HashMap(); persistent = new HashSet(); - try { - enabled = !"false".equals(VDL2Config.getConfig().getProperty(VDL2ConfigProperties.FILE_GC_ENABLED)); - } - catch (IOException e) { - //enabled by default - enabled = true; - } + enabled = SwiftConfig.getDefault().isFileGCEnabled(); if (enabled) { thread = new Thread(this, "File Garbage Collector"); thread.setDaemon(true); Modified: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java =================================================================== --- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -20,7 +20,6 @@ */ package org.griphyn.vdl.mapping.nodes; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -44,7 +43,7 @@ import org.griphyn.vdl.mapping.PhysicalFormat; import org.griphyn.vdl.type.Field; import org.griphyn.vdl.type.Type; -import org.griphyn.vdl.util.VDL2Config; +import org.griphyn.vdl.util.SwiftConfig; @@ -79,11 +78,7 @@ public static boolean provenance = false; static { - try { - provenance = VDL2Config.getConfig().getProvenanceLog(); - } - catch (IOException e) { - } + provenance = SwiftConfig.getDefault().isProvenanceEnabled(); } protected Field field; Modified: trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java =================================================================== --- trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -10,25 +10,46 @@ package org.griphyn.vdl.util; import java.io.File; +import java.util.Arrays; import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.SortedSet; import java.util.TreeSet; -public abstract class ConfigPropertyType { - public static final ConfigPropertyType BOOLEAN = choices("true", "false"); - public static final ConfigPropertyType ONOFF = choices("on", "off"); - public static final ConfigPropertyType STRING = new CPTString(); - public static final ConfigPropertyType INT = new Int(); - public static final ConfigPropertyType FLOAT = new CPTFloat(); - public static final ConfigPropertyType FILE = new CPTFile(); +import org.globus.cog.abstraction.impl.common.execution.WallTime; +import org.globus.swift.catalog.types.Arch; +import org.globus.swift.catalog.types.Os; + +public abstract class ConfigPropertyType { + public static final ConfigPropertyType BOOLEAN = new CPTBoolean(); + public static final ConfigPropertyType STRING = new CPTString(); + public static final ConfigPropertyType INT = new Int(); + public static final ConfigPropertyType THROTTLE = new Throttle(); + public static final ConfigPropertyType PORT_RANGE = new PortRange(); + public static final ConfigPropertyType STRICTLY_POSITIVE_INT = new SPInt(); + public static final ConfigPropertyType POSITIVE_INT = new PInt(); + public static final ConfigPropertyType POSITIVE_FLOAT = new PFloat(); + public static final ConfigPropertyType FLOAT = new CPTFloat(); + public static final ConfigPropertyType FILE = new CPTFile(); + public static final ConfigPropertyType TIME = new CPTTime(); + public static final ConfigPropertyType OBJECT = new CPTObject(); + public static final ConfigPropertyType STRING_LIST = new StringList(); + public static final ConfigPropertyType OS = new CPTOS(); - public static ConfigPropertyType choices(String... values) { + public static ConfigPropertyType choices(String... values) { return new Choices(values); } - public abstract void checkValue(String propName, String value, String source); + @SuppressWarnings("unchecked") + public Object check(String propName, Object value, String source) { + return checkValue(propName, (T) value, source); + } + public abstract Object checkValue(String propName, T value, String source); + + public abstract ConfigPropertyType getBaseType(); + private static String pp(Collection c) { StringBuilder sb = new StringBuilder(); Iterator i = c.iterator(); @@ -43,8 +64,8 @@ return sb.toString(); } - private static class Choices extends ConfigPropertyType { - private SortedSet choices; + public static class Choices extends ConfigPropertyType { + protected SortedSet choices; public Choices(String... values) { choices = new TreeSet(); @@ -54,55 +75,343 @@ } @Override - public void checkValue(String propName, String value, String source) { + public Object checkValue(String propName, String value, String source) { if (!choices.contains(value)) { throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + propName + "'. Valid values are: " + pp(choices)); } + return value; } + + @Override + public ConfigPropertyType getBaseType() { + return STRING; + } + + @Override + public String toString() { + return "one of " + choices; + } } - private static class CPTString extends ConfigPropertyType { + private static class CPTOS extends Choices { + public CPTOS() { + super(); + int ix = 0; + for (Arch a : Arch.values()) { + for (Os o : Os.values()) { + choices.add(a + "::" + o); + } + } + } + } + + private static class CPTString extends ConfigPropertyType { @Override - public void checkValue(String propName, String value, String source) { + public Object checkValue(String propName, String value, String source) { // all values accepted + return value; } + + @Override + public ConfigPropertyType getBaseType() { + return STRING; + } + + @Override + public String toString() { + return "string"; + } } - private static class Int extends ConfigPropertyType { + private static class Int extends ConfigPropertyType { @Override - public void checkValue(String propName, String value, String source) { - try { - Integer.parseInt(value); + public Object checkValue(String propName, Integer value, String source) { + return value; + } + + @Override + public ConfigPropertyType getBaseType() { + return INT; + } + + @Override + public String toString() { + return "integer"; + } + } + + private static class CPTBoolean extends ConfigPropertyType { + @Override + public Object checkValue(String propName, Boolean value, String source) { + return value; + } + + @Override + public ConfigPropertyType getBaseType() { + return BOOLEAN; + } + + @Override + public String toString() { + return "boolean"; + } + } + + private static class SPInt extends ConfigPropertyType { + @Override + public Object checkValue(String propName, Integer value, String source) { + if (value <= 0) { + throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + + propName + "'. Must be a " + toString()); } - catch (NumberFormatException e) { + return value; + } + + @Override + public ConfigPropertyType getBaseType() { + return INT; + } + + @Override + public String toString() { + return "strictly positive integer"; + } + } + + private static class PInt extends ConfigPropertyType { + @Override + public Object checkValue(String propName, Integer value, String source) { + if (value < 0) { throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + - propName + "'. Must be an integer"); + propName + "'. Must be a " + toString()); } + return value; } + + @Override + public ConfigPropertyType getBaseType() { + return INT; + } + + @Override + public String toString() { + return "positive integer"; + } } + - private static class CPTFloat extends ConfigPropertyType { + private static class Throttle extends ConfigPropertyType { @Override - public void checkValue(String propName, String value, String source) { - try { - Double.parseDouble(value); + public Object checkValue(String propName, Object value, String source) { + if ("off".equals(value)) { + return Integer.MAX_VALUE; } - catch (NumberFormatException e) { + else if (value instanceof Integer) { + Integer i = (Integer) value; + if (i > 0) { + return i; + } + } + throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + + propName + "'. Must be an " + toString()); + } + + @Override + public ConfigPropertyType getBaseType() { + return INT; + } + + @Override + public String toString() { + return "integer greater than zero or \"off\""; + } + } + + private static class PFloat extends ConfigPropertyType { + @Override + public Object checkValue(String propName, Double value, String source) { + if (value < 0) { throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + - propName + "'. Must be a floating point number."); + propName + "'. Must be a " + toString()); } + return value; } + + @Override + public ConfigPropertyType getBaseType() { + return FLOAT; + } + + @Override + public String toString() { + return "positive number"; + } } - private static class CPTFile extends ConfigPropertyType { + private static class CPTFloat extends ConfigPropertyType { @Override - public void checkValue(String propName, String value, String source) { + public Object checkValue(String propName, Double value, String source) { + return value; + } + + @Override + public ConfigPropertyType getBaseType() { + return FLOAT; + } + } + + public static class Interval extends ConfigPropertyType { + private double l, h; + + public Interval(double l, double h) { + this.l = l; + this.h = h; + } + + @Override + public Object checkValue(String propName, Double value, String source) { + if (value < l || value > h) { + throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + + propName + "'. Must be a " + toString()); + } + return value; + } + + @Override + public ConfigPropertyType getBaseType() { + return FLOAT; + } + + @Override + public String toString() { + return "floating point number in the interval [" + l + ", " + h + "]"; + } + } + + private static class CPTTime extends ConfigPropertyType { + @Override + public Object checkValue(String propName, String value, String source) { + try { + WallTime.timeToSeconds(value); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException(source + ":\n\tInvalid time value '" + value + "' for property '" + + propName + "'. Mist be a " + toString()); + } + return value; + } + + @Override + public ConfigPropertyType getBaseType() { + return STRING; + } + + @Override + public String toString() { + return "string in one of the formats MM, HH:MM, or HH:MM:SS"; + } + } + + private static class PortRange extends ConfigPropertyType { + @Override + public Object checkValue(String propName, String value, String source) { + String[] els = value.split(",\\s*"); + if (els.length == 2) { + try { + Integer.parseInt(els[0]); + Integer.parseInt(els[1]); + return value; + } + catch (NumberFormatException e) { + } + } + throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + + propName + "'. Must be a " + toString()); + } + + @Override + public ConfigPropertyType getBaseType() { + return STRING; + } + + @Override + public String toString() { + return "port range in the format 'port1, port2'"; + } + } + + private static class CPTFile extends ConfigPropertyType { + @Override + public Object checkValue(String propName, String value, String source) { File f = new File(value); if (!f.exists()) { throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + propName + "'. File does not exist."); } + return value; } + + @Override + public ConfigPropertyType getBaseType() { + return STRING; + } + + @Override + public String toString() { + return "file path"; + } } + + private static class CPTObject extends ConfigPropertyType { + @Override + public Object checkValue(String propName, Object value, String source) { + return value; + } + + + @Override + public ConfigPropertyType getBaseType() { + return OBJECT; + } + + @Override + public String toString() { + return "object"; + } + } + + private static class StringList extends ConfigPropertyType { + @Override + public Object checkValue(String propName, Object value, String source) { + if (value instanceof List) { + List l = (List) value; + boolean allStrings = true; + for (Object o : l) { + if (!(o instanceof String)) { + allStrings = false; + } + } + if (allStrings) { + return value; + } + } + else if (value instanceof String) { + // also allow comma separated strings in a string + return Arrays.asList(((String) value).split(",\\s*")); + } + throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" + + propName + "'. Must be a " + toString()); + } + + @Override + public ConfigPropertyType getBaseType() { + return OBJECT; + } + + @Override + public String toString() { + return "list of strings"; + } + } } Added: trunk/src/org/griphyn/vdl/util/ConfigTree.java =================================================================== --- trunk/src/org/griphyn/vdl/util/ConfigTree.java (rev 0) +++ trunk/src/org/griphyn/vdl/util/ConfigTree.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -0,0 +1,319 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 6, 2014 + */ +package org.griphyn.vdl.util; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; + +public class ConfigTree { + + public static class Node { + private Map> nodes; + private T value; + + protected void checkEmpty(String k) { + if (k.isEmpty()) { + throw new IllegalArgumentException(); + } + } + + protected String first(String k) { + int ix = k.indexOf('.'); + if (ix == -1) { + return k; + } + else { + return k.substring(0, ix); + } + } + + protected String rest(String k) { + int ix = k.indexOf('.'); + if (ix == -1) { + return ""; + } + else { + return k.substring(ix + 1); + } + } + + public Object get() { + return value; + } + + public T get(String k) { + return get(k, null); + } + + public T get(String k, String wildcard) { + if (k.isEmpty()) { + return value; + } + if (nodes == null || nodes.isEmpty()) { + throw new NoSuchElementException(); + } + Node t = nodes.get(first(k)); + if (t == null && wildcard != null) { + t = nodes.get(wildcard); + } + if (t == null) { + throw new NoSuchElementException(); + } + return t.get(rest(k), wildcard); + } + + public boolean hasKey(String k) { + if (k.isEmpty()) { + return value != null; + } + if (nodes == null || nodes.isEmpty()) { + return false; + } + Node t = nodes.get(first(k)); + if (t == null) { + return false; + } + return t.hasKey(rest(k)); + } + + public T put(String k, T v) { + if (k.isEmpty()) { + return set(v); + } + else { + if (nodes == null) { + nodes = new HashMap>(); + } + String first = first(k); + if (k == first) { + return getOrCreateTree(k).set(v); + } + else { + return getOrCreateTree(first).put(rest(k), v); + } + } + } + + private Node getOrCreateTree(String k) { + Node t = nodes.get(k); + if (t == null) { + t = new Node(); + nodes.put(k, t); + return t; + } + else { + return t; + } + } + + public void getLeafPaths(List l, String partial) { + if (value != null) { + l.add(partial); + } + if (nodes != null) { + for (Map.Entry> e : nodes.entrySet()) { + if (partial == null) { + e.getValue().getLeafPaths(l, e.getKey()); + } + else { + e.getValue().getLeafPaths(l, partial + "." + e.getKey()); + } + } + } + } + + public void expandWildcards(List l, String k, String wildcard, String partial) { + if (value != null) { + if (partial == null) { + l.add(k); + } + else { + l.add(partial + "." + k); + } + } + if (nodes == null || nodes.isEmpty()) { + if (rest(k).isEmpty()) { + if (partial == null) { + l.add(k); + } + else { + l.add(partial + "." + k); + } + } + return; + } + String mk = first(k); + if (mk.equals(wildcard)) { + for (Map.Entry> e : nodes.entrySet()) { + Node n = e.getValue(); + String rest = rest(k); + String p; + if (partial == null) { + p = e.getKey(); + } + else { + p = partial + "." + e.getKey(); + } + n.expandWildcards(l, rest, wildcard, p); + } + } + else { + Node t = nodes.get(mk); + if (t == null) { + if (first(rest(k)).equals(wildcard)) { + // x.* is allowed to not be there + return; + } + if (partial == null) { + l.add(k); + } + else { + l.add(partial + "." + k); + } + return; + } + String rest = rest(k); + String p; + if (partial == null) { + p = mk; + } + else { + p = partial + "." + mk; + } + t.expandWildcards(l, rest, wildcard, p); + } + } + + public T set(T v) { + T old = value; + value = v; + return old; + } + + public Set>> entrySet() { + if (nodes == null) { + Map> empty = Collections.emptyMap(); + return empty.entrySet(); + } + else { + return nodes.entrySet(); + } + } + + private void toString(StringBuilder sb, int level, String k) { + for (int i = 0; i < level; i++) { + sb.append('\t'); + } + if (nodes == null || nodes.isEmpty()) { + if (value != null) { + sb.append(k); + sb.append(": "); + if (value instanceof String) { + sb.append('\"'); + sb.append(value); + sb.append('\"'); + } + else { + sb.append(value); + } + sb.append('\n'); + } + } + else if (nodes.size() == 1) { + String key = nodes.keySet().iterator().next(); + if (k == null) { + nodes.values().iterator().next().toString(sb, 0, key); + } + else { + nodes.values().iterator().next().toString(sb, 0, k + "." + key); + } + } + else { + if (k != null) { + sb.append(k); + sb.append(' '); + } + sb.append("{\n"); + for (Map.Entry> e : nodes.entrySet()) { + e.getValue().toString(sb, level + 1, e.getKey()); + } + for (int i = 0; i < level; i++) { + sb.append('\t'); + } + sb.append("}\n"); + } + } + } + + private Node root; + + public ConfigTree() { + root = new Node(); + } + + public T get(String k) { + return get(k, null); + } + + public T put(String k, T v) { + return root.put(k, v); + } + + public T get(String k, String wildcard) { + try { + return root.get(k, wildcard); + } + catch (IllegalArgumentException e) { + throw new NoSuchElementException("Not a leaf: " + k); + } + catch (NoSuchElementException e) { + return null; + } + } + + public List getLeafPaths() { + List l = new ArrayList(); + root.getLeafPaths(l, null); + return l; + } + + /** + * Find all paths matching the given path. Wildcards are expanded based + * on what's in the tree, but the full paths do not need to exist in the tree. + * + * So if a.1.b.2 and a.1.b.3 were in the tree, a.*.b.*.c would generate + * a.1.b.2.c and a.1.b.3.c + * + */ + public List expandWildcards(String key, String wildcard) { + List l = new ArrayList(); + root.expandWildcards(l, key, wildcard, null); + return l; + } + + public boolean hasKey(String k) { + return root.hasKey(k); + } + + public Set>> entrySet() { + return root.entrySet(); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + root.toString(sb, 0, null); + return sb.toString(); + } +} Added: trunk/src/org/griphyn/vdl/util/ConvertConfig.java =================================================================== --- trunk/src/org/griphyn/vdl/util/ConvertConfig.java (rev 0) +++ trunk/src/org/griphyn/vdl/util/ConvertConfig.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -0,0 +1,772 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jan 7, 2013 + */ +package org.griphyn.vdl.util; + +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.globus.cog.abstraction.impl.common.AbstractionFactory; +import org.globus.cog.abstraction.impl.common.ProviderMethodException; +import org.globus.cog.abstraction.impl.common.execution.WallTime; +import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl; +import org.globus.cog.abstraction.impl.common.task.InvalidProviderException; +import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl; +import org.globus.cog.abstraction.impl.common.task.ServiceImpl; +import org.globus.cog.abstraction.interfaces.ExecutionService; +import org.globus.cog.abstraction.interfaces.Service; +import org.globus.cog.abstraction.interfaces.ServiceContact; +import org.globus.cog.karajan.scheduler.WeightedHost; +import org.globus.cog.karajan.util.BoundContact; +import org.globus.cog.karajan.util.ContactSet; +import org.globus.cog.karajan.util.TypeUtil; +import org.globus.cog.util.ArgumentParser; +import org.globus.cog.util.ArgumentParserException; +import org.globus.swift.catalog.TCEntry; +import org.globus.swift.catalog.TransformationCatalog; +import org.globus.swift.catalog.site.SiteCatalogParser; +import org.globus.swift.catalog.transformation.File; +import org.globus.swift.catalog.util.Profile; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +public class ConvertConfig { + + private enum Version { + V1, V2; + } + + private ContactSet parseSites(String fileName) { + SiteCatalogParser p = new SiteCatalogParser(fileName); + try { + Document doc = p.parse(); + return buildResources(doc); + } + catch (Exception e) { + throw new RuntimeException("Failed to parse site catalog", e); + } + } + + private TransformationCatalog loadTC(String fileName) { + return File.getNonSingletonInstance(fileName); + } + + public void run(String sitesFile, String tcFile, String conf, PrintStream ps) { + if (sitesFile == null && tcFile == null && conf == null) { + throw new IllegalArgumentException("You must specify at " + + "least one file type to convert"); + } + ContactSet sites = null; + if (sitesFile != null) { + sites = parseSites(sitesFile); + } + TransformationCatalog tc = null; + if (tcFile != null) { + tc = loadTC(tcFile); + } + + generateConfig(sites, tc, ps); + + if (conf != null) { + Properties props = new Properties(); + try { + props.load(new FileReader(conf)); + generateConfig(props, ps); + } + catch (IOException e) { + throw new RuntimeException("Could not load config file", e); + } + } + } + + private void generateConfig(Properties props, PrintStream ps) { + Set processed = new HashSet(); + for (String name : propOrder) { + if (props.containsKey(name)) { + prop(name, props.get(name), ps); + processed.add(name); + } + } + for (Map.Entry e : props.entrySet()) { + String name = (String) e.getKey(); + if (!processed.contains(name)) { + prop(name, props.get(name), ps); + } + } + } + + private void prop(String name, Object value, PrintStream ps) { + if (removedAttrs.contains(name)) { + ps.println("# option removed: " + name); + return; + } + if (name.equals("use.provider.staging")) { + if (TypeUtil.toBoolean(value)) { + ps.println("staging: \"provider\""); + } + return; + } + if (name.equals("use.wrapper.staging")) { + if (TypeUtil.toBoolean(value)) { + ps.println("staging: \"wrapper\""); + } + return; + } + Attr a = props.get(name); + if (a == null) { + throw new RuntimeException("Unknown configuration property '" + name + "'"); + } + ps.print(a.name + ": "); + printValue(a.type, value, ps); + } + + private void generateConfig(ContactSet sites, TransformationCatalog tc, PrintStream ps) { + ps.println("include \"${swift.home}/etc/swift.conf\""); + ps.println(); + + if (sites != null) { + for (BoundContact bc : sites.getContacts()) { + ps.println("site." + bc.getName() + " {"); + boolean hasFileService = false; + boolean hasCoasterService = false; + for (Map.Entry e : bc.getServices().entrySet()) { + String provider = e.getKey().provider; + if (provider == null) { + continue; + } + if (e.getKey().type == Service.EXECUTION) { + ExecutionService es = (ExecutionService) e.getValue(); + ps.println("\texecution {"); + ps.println("\t\ttype: \"" + provider + "\""); + if (es.getServiceContact() != null) { + ps.println("\t\tURL: \"" + es.getServiceContact().getContact() + "\""); + } + if (es.getJobManager() != null) { + ps.println("\t\tjobManager: \"" + es.getJobManager() + "\""); + } + if (provider.equals("coaster")) { + hasCoasterService = true; + generateCoasterServiceAttributes(bc, ps); + } + ps.println("\t}"); + if (!provider.equals("coaster")) { + generateServiceAttributes(es, ps); + } + } + else if (e.getKey().type == Service.FILE_OPERATION) { + hasFileService = true; + ps.println("\tfilesystem {"); + ps.println("\t\ttype: \"" + provider + "\""); + if (e.getValue().getServiceContact() != null) { + ps.println("\t\tURL: \"" + e.getValue().getServiceContact().getContact() + "\""); + } + ps.println("\t}"); + } + } + if (!hasFileService) { + ps.println("\tstaging: \"provider\""); + } + if (bc.hasProperty("workdir")) { + ps.println("\tworkDirectory: \"" + bc.getProperty("workdir") + "\""); + } + if (bc.hasProperty("scratch")) { + ps.println("\tscratch: \"" + bc.getProperty("scratch") + "\""); + } + generateSiteAttributes(bc.getProperties(), ps); + if (tc != null) { + generateAppInfo(bc.getName(), tc, hasCoasterService, 1, ps); + } + ps.println("}\n"); + } + } + if (tc != null) { + generateAppInfo("*", tc, false, 0, ps); + } + } + + private void generateAppInfo(String host, TransformationCatalog tc, boolean hasCoasterService, int level, PrintStream ps) { + try { + for (TCEntry e : tc.getTC()) { + if (e.getResourceId().equals(host)) { + if (e.getLogicalName().equals("*")) { + tabs(level, ps); + ps.println("app.ALL {"); + } + else { + tabs(level, ps); + ps.println("app." + e.getLogicalName() + " {"); + } + tabs(level + 1, ps); + ps.println("executable: " + e.getPhysicalTransformation()); + if (e.getProfiles() != null) { + generateAppProfiles(e, hasCoasterService, level, ps); + } + tabs(level, ps); + ps.println("}\n"); + } + } + } + catch (Exception e) { + throw new RuntimeException("Failed to get TC data", e); + } + } + + private void generateAppProfiles(TCEntry e, boolean hasCoasterService, int level, PrintStream ps) { + StringBuilder options = new StringBuilder(); + for (Profile p : e.getProfiles()) { + if (p.getProfileKey().startsWith("env.")) { + tabs(level, ps); + ps.println(p.getProfileKey() + ": \"" + p.getProfileValue() + "\""); + } + else if (p.getProfileKey().equals("maxwalltime")) { + tabs(level, ps); + ps.println("maxWallTime: \"" + p.getProfileValue() + "\""); + } + else if (p.getProfileKey().equals("queue")) { + if (!hasCoasterService) { + tabs(level, ps); + ps.println("jobQueue: \"" + p.getProfileValue() + "\""); + } + } + else if (p.getProfileKey().equals("project")) { + if (!hasCoasterService) { + tabs(level, ps); + ps.println("jobProject: \"" + p.getProfileValue() + "\""); + } + } + else { + if (options.length() != 0) { + options.append(", "); + } + else { + options.append("{"); + } + options.append(p.getProfileKey() + ": \"" + p.getProfileValue() + "\""); + } + } + if (options.length() != 0) { + options.append("}"); + tabs(level, ps); + ps.println("options: " + options.toString()); + } + ps.println("\t}"); + } + + private void tabs(int count, PrintStream ps) { + for (int i = 0; i < count; i++) { + ps.print('\t'); + } + } + + private void generateSiteAttributes(Map properties, PrintStream ps) { + if (properties.containsKey("sysinfo")) { + ps.println("\tOS: \"" + properties.get("sysinfo") + "\""); + } + if (properties.containsKey("delayBase")) { + ps.println("\tdelayBase: " + properties.get("delayBase")); + } + if (properties.containsKey("initialScore")) { + double jobThrottle = 2; + if (properties.containsKey("jobThrottle")) { + jobThrottle = TypeUtil.toDouble(properties.get("jobThrottle")); + } + double initialScore = TypeUtil.toDouble(properties.get("initialScore")); + ps.println("\tmaxParallelTasks: " + (int) (jobThrottle * WeightedHost.T + 1)); + ps.println("\tinitialParallelTasks: " + (int) (jobThrottle * WeightedHost.computeTScore(initialScore) + 1)); + } + if (properties.containsKey("maxSubmitRate")) { + ps.println("\tmaxSubmitRate: " + properties.get("maxSubmitRate")); + } + } + + private enum AttrType { + INT, FLOAT, STRING, BOOLEAN, TIME_FROM_SECONDS; + } + + private static class Attr { + public Attr(String k, AttrType t) { + this.name = k; + this.type = t; + } + public AttrType type; + public String name; + } + + private static final Set removedAttrs; + private static final Map coasterAttrs; + private static final Map serviceAttrs; + private static final Map props; + private static List propOrder; + + static { + removedAttrs = new HashSet(); + removedAttrs.add("pgraph"); + removedAttrs.add("pgraph.graph.options"); + removedAttrs.add("pgraph.node.options"); + removedAttrs.add("clustering.enabled"); + removedAttrs.add("clustering.queue.delay"); + removedAttrs.add("clustering.min.time"); + removedAttrs.add("kickstart.enabled"); + removedAttrs.add("kickstart.always.transfer"); + removedAttrs.add("sites.file"); + removedAttrs.add("tc.file"); + + coasterAttrs = new HashMap(); + attr(coasterAttrs, "queue", "jobQueue", AttrType.STRING); + attr(coasterAttrs, "project", "jobProject", AttrType.STRING); + attr(coasterAttrs, "maxtime", "jobMaxTime", AttrType.TIME_FROM_SECONDS); + attr(coasterAttrs, "reserve", AttrType.STRING); + attr(coasterAttrs, "lowOverallocation", AttrType.INT); + attr(coasterAttrs, "highOverallocation", AttrType.INT); + attr(coasterAttrs, "slots", "maxJobs", AttrType.INT); + attr(coasterAttrs, "maxNodes", "maxNodesPerJob", AttrType.INT); + attr(coasterAttrs, "overallocationDecayFactor", AttrType.FLOAT); + attr(coasterAttrs, "internalHostname", AttrType.STRING); + attr(coasterAttrs, "allocationStepSize", AttrType.FLOAT); + attr(coasterAttrs, "nodeGranularity", AttrType.INT); + attr(coasterAttrs, "remoteMonitorEnabled", AttrType.BOOLEAN); + attr(coasterAttrs, "userHomeOverride", AttrType.STRING); + attr(coasterAttrs, "workerLoggingLevel", AttrType.STRING); + attr(coasterAttrs, "workerLoggingDirectory", AttrType.STRING); + attr(coasterAttrs, "workerManager", AttrType.STRING); + attr(coasterAttrs, "softImage", AttrType.STRING); + attr(coasterAttrs, "jobsPerNode", "tasksPerNode", AttrType.INT); + + serviceAttrs = new HashMap(); + attr(serviceAttrs, "queue", "jobQueue", AttrType.STRING); + attr(serviceAttrs, "project", "jobProject", AttrType.STRING); + attr(serviceAttrs, "maxtime", "jobMaxTime", AttrType.STRING); + attr(serviceAttrs, "jobType", "jobType", AttrType.STRING); + + props = new HashMap(); + propOrder = new ArrayList(); + prop("hostname", "hostName", AttrType.STRING); + prop("tcp.port.range", "TCPPortRange", AttrType.STRING); + prop("lazy.errors", "lazyErrors", AttrType.BOOLEAN); + prop("execution.retries", "executionRetries", AttrType.INT); + prop("caching.algorithm", "cachingAlgorithm", AttrType.STRING); + prop("throttle.submit", "jobSubmitThrottle", AttrType.INT); + prop("throttle.host.submit", "hostJobSubmitThrottle", AttrType.INT); + prop("throttle.transfers", "fileTransfersThrottle", AttrType.INT); + prop("throttle.file.operations", "fileOperationsThrottle", AttrType.INT); + prop("throttle.score.job.factor", "siteScoreThrottlingFactor", AttrType.INT); + prop("sitedir.keep", "keepSiteDir", AttrType.BOOLEAN); + prop("provenance.log", "logProvenance", AttrType.BOOLEAN); + prop("replication.enabled", "replicationEnabled", AttrType.BOOLEAN); + prop("replication.min.queue.time", "replicationMinQueueTime", AttrType.INT); + prop("replication.limit", "replicationLimit", AttrType.INT); + prop("status.mode", "statusMode", AttrType.STRING); + prop("wrapper.parameter.mode", "wrapperParameterMode", AttrType.STRING); + prop("wrapper.invocation.mode", "wrapperInvocationMode", AttrType.STRING); + prop("cdm.broadcast.mode", "CDMBroadcastMode", AttrType.STRING); + prop("provider.staging.pin.swiftfiles", "providerStagingPinSwiftFiles", AttrType.BOOLEAN); + prop("ticker.date.format", "tickerDateFormat", AttrType.STRING); + prop("ticker.prefix", "tickerPrefix", AttrType.STRING); + prop("ticker.enabled", "tickerEnabled", AttrType.BOOLEAN); + prop("file.gc.enabled", "fileGCEnabled", AttrType.BOOLEAN); + prop("mapping.checker", "mappingCheckerEnabled", AttrType.BOOLEAN); + prop("wrapperlog.always.transfer", "alwaysTransferWrapperLog", AttrType.BOOLEAN); + prop("tracing.enabled", "tracingEnabled", AttrType.BOOLEAN); + prop("wrapper.staging.local.server", "wrapperStagingLocalServer", AttrType.STRING); + prop("foreach.max.threads", "maxForeachThreads", AttrType.INT); + } + + + private static void attr(Map m, String k, AttrType t) { + m.put("globus:" + k.toLowerCase(), new Attr(k, t)); + } + + private static void attr(Map m, String k, String v, AttrType t) { + m.put("globus:" + k.toLowerCase(), new Attr(v, t)); + } + + private static void prop(String k, AttrType t) { + if (!SwiftConfig.SCHEMA.isNameValid(k)) { + throw new Error("Invalid prop: '" + k + "'"); + } + props.put(k, new Attr(k, t)); + propOrder.add(k); + } + + private static void prop(String k, String v, AttrType t) { + if (!SwiftConfig.SCHEMA.isNameValid(v)) { + throw new Error("Invalid prop: '" + v + "'"); + } + props.put(k, new Attr(v, t)); + propOrder.add(k); + } + + private void generateCoasterServiceAttributes(BoundContact bc, PrintStream ps) { + for (String attr : bc.getProperties().keySet()) { + Attr a = coasterAttrs.get(attr.toLowerCase()); + if (a != null) { + ps.print("\t\t" + a.name + ": "); + printValue(a.type, bc.getProperty(attr), ps); + } + else { + if (attr.startsWith("globus.")) { + throw new RuntimeException("Unrecognize profile: '" + attr + "'"); + } + } + } + } + + private void printValue(AttrType type, Object value, PrintStream ps) { + switch (type) { + case INT: + case FLOAT: + case BOOLEAN: + ps.println(value); + break; + case STRING: + ps.println("\"" + value + "\""); + break; + case TIME_FROM_SECONDS: + ps.println("\"" + WallTime.format("hms", TypeUtil.toInt(value)) + "\""); + break; + } + } + + private void generateServiceAttributes(Service s, PrintStream ps) { + for (String attr : s.getAttributeNames()) { + Attr a = serviceAttrs.get(attr); + if (a != null) { + ps.print("\t\t" + a.name + ": "); + switch (a.type) { + case INT: + case FLOAT: + case BOOLEAN: + ps.println(s.getAttribute(attr)); + break; + case STRING: + ps.println("\"" + s.getAttribute(attr) + "\""); + break; + } + } + else { + if (attr.startsWith("globus.") && coasterAttrs.containsKey(attr)) { + throw new RuntimeException("Unrecognize profile: '" + attr + "'"); + } + } + } + } + + private ContactSet buildResources(Document doc) { + Node root = getRoot(doc); + + if (root.getLocalName().equals("config")) { + return parse(root, Version.V1); + } + else if (root.getLocalName().equals("sites")) { + return parse(root, Version.V1); + } + else { + throw new IllegalArgumentException("Illegal sites file root node: " + root.getLocalName()); + } + } + + + private ContactSet parse(Node config, Version v) { + ContactSet cs = new ContactSet(); + NodeList pools = config.getChildNodes(); + for (int i = 0; i < pools.getLength(); i++) { + Node n = pools.item(i); + if (n.getNodeType() == Node.ELEMENT_NODE) { + try { + BoundContact bc = pool(n, v); + if (bc != null) { + cs.addContact(bc); + } + } + catch (Exception e) { + throw new RuntimeException("Invalid site entry '" + poolName(n, v) + "': ", e); + } + } + } + return cs; + } + + private String poolName(Node site, Version v) { + if (site.getLocalName().equals("pool")) { + return attr(site, "handle"); + } + else if (site.getLocalName().equals("site")) { + return attr(site, "name"); + } + else { + throw new IllegalArgumentException("Invalid node: " + site.getLocalName()); + } + } + + private Node getRoot(Document doc) { + NodeList l = doc.getChildNodes(); + for (int i = 0; i < l.getLength(); i++) { + if (l.item(i).getNodeType() == Node.ELEMENT_NODE) { + return l.item(i); + } + } + throw new IllegalArgumentException("Missing root element"); + } + + private BoundContact pool(Node n, Version v) throws InvalidProviderException, ProviderMethodException { + if (n.getNodeType() != Node.ELEMENT_NODE) { + return null; + } + String name = poolName(n, v); + BoundContact bc = new BoundContact(name); + + String sysinfo = attr(n, "sysinfo", null); + if (sysinfo != null) { + bc.setProperty("sysinfo", sysinfo); + } + + NodeList cs = n.getChildNodes(); + + for (int i = 0; i < cs.getLength(); i++) { + Node c = cs.item(i); + if (c.getNodeType() != Node.ELEMENT_NODE) { + continue; + } + String ctype = c.getNodeName(); + + if (v == Version.V1 && ctype.equals("gridftp")) { + bc.addService(gridftp(c)); + } + else if (v == Version.V1 && ctype.equals("jobmanager")) { + bc.addService(jobmanager(c)); + } + else if (ctype.equals("execution")) { + bc.addService(execution(c)); + } + else if (ctype.equals("filesystem")) { + bc.addService(filesystem(c)); + } + else if (ctype.equals("workdirectory")) { + bc.setProperty("workdir", text(c)); + } + else if (ctype.equals("scratch")) { + bc.setProperty("scratch", text(c)); + } + else if (ctype.equals("env")) { + env(bc, c); + } + else if (ctype.equals("profile")) { + profile(bc, c); + } + else { + throw new IllegalArgumentException("Unknown node type: " + ctype); + } + } + return bc; + } + + private Service jobmanager(Node n) throws InvalidProviderException, ProviderMethodException { + String provider; + String url = attr(n, "url"); + String major = attr(n, "major"); + if (url.equals("local://localhost")) { + provider = "local"; + } + else if (url.equals("pbs://localhost")) { + provider = "pbs"; + } + else if ("2".equals(major)) { + provider = "gt2"; + } + else if ("4".equals(major)) { + provider = "gt4"; + } + else { + throw new IllegalArgumentException("Unknown job manager version: " + major + ", url = '" + url + "'"); + } + + ServiceContact contact = new ServiceContactImpl(url); + return new ServiceImpl(provider, Service.EXECUTION, + contact, AbstractionFactory.newSecurityContext(provider, contact)); + } + + private Service gridftp(Node n) throws InvalidProviderException, ProviderMethodException { + String url = attr(n, "url"); + if (url.equals("local://localhost")) { + return new ServiceImpl("local", Service.FILE_OPERATION, new ServiceContactImpl("localhost"), null); + } + else { + ServiceContact contact = new ServiceContactImpl(url); + return new ServiceImpl("gsiftp", Service.FILE_OPERATION, + contact, AbstractionFactory.newSecurityContext("gsiftp", contact)); + } + } + + private Service execution(Node n) throws InvalidProviderException, ProviderMethodException { + String provider = attr(n, "provider"); + String url = attr(n, "url", null); + String jobManager = attr(n, "jobManager", null); + if (jobManager == null) { + jobManager = attr(n, "jobmanager", null); + } + + ExecutionService s = new ExecutionServiceImpl(); + s.setProvider(provider); + ServiceContact contact = null; + if (url != null) { + contact = new ServiceContactImpl(url); + s.setServiceContact(contact); + s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact)); + } + + if (jobManager != null) { + s.setJobManager(jobManager); + } + + return s; + } + + private Service filesystem(Node n) throws InvalidProviderException, ProviderMethodException { + String provider = attr(n, "provider"); + String url = attr(n, "url", null); + + Service s = new ServiceImpl(); + s.setType(Service.FILE_OPERATION); + s.setProvider(provider); + + ServiceContact contact = null; + if (url != null) { + contact = new ServiceContactImpl(url); + s.setServiceContact(contact); + s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact)); + } + + return s; + } + + private void env(BoundContact bc, Node n) { + String key = attr(n, "key"); + String value = text(n); + + bc.setProperty("env:" + key, value); + } + + private void profile(BoundContact bc, Node n) { + String ns = attr(n, "namespace"); + String key = attr(n, "key"); + String value = text(n); + + if (value == null) { + throw new IllegalArgumentException("No value for profile " + ns + ":" + key); + } + if (ns.equals("karajan")) { + bc.setProperty(key, value); + } + else { + bc.setProperty(ns + ":" + key, value); + } + } + + private String text(Node n) { + if (n.getFirstChild() != null) { + return n.getFirstChild().getNodeValue(); + } + else { + return null; + } + } + + private String attr(Node n, String name) { + NamedNodeMap attrs = n.getAttributes(); + if (attrs != null) { + Node attr = attrs.getNamedItem(name); + if (attr == null) { + throw new IllegalArgumentException("Missing " + name); + } + else { + return attr.getNodeValue(); + } + } + else { + throw new IllegalArgumentException("Missing " + name); + } + } + + private String attr(Node n, String name, String defVal) { + NamedNodeMap attrs = n.getAttributes(); + if (attrs != null) { + Node attr = attrs.getNamedItem(name); + if (attr == null) { + return defVal; + } + else { + return attr.getNodeValue(); + } + } + else { + return defVal; + } + } + + public static void main(String[] args) { + ArgumentParser ap = new ArgumentParser(); + ap.setExecutableName("swift-convert-config"); + ap.addFlag("help", "Displays usage information"); + ap.addOption("sites.file", "Specifies a sites file to convert", + "file", ArgumentParser.OPTIONAL); + ap.addOption("tc.file", "Specifies a tc.data file to convert", + "file", ArgumentParser.OPTIONAL); + ap.addOption("config", "Specifies an old Swift configuration file to convert", + "file", ArgumentParser.OPTIONAL); + ap.addOption("out", "Indicates that the output should go to a file instead of standard output", + "file", ArgumentParser.OPTIONAL); + + try { + ap.parse(args); + } + catch (ArgumentParserException e) { + System.out.println("Error parsing command line arguments: " + e.getMessage()); + usage(ap); + System.exit(1); + } + if (ap.isPresent("help")) { + usage(ap); + System.exit(0); + } + ConvertConfig cc = new ConvertConfig(); + try { + PrintStream ps = System.out; + if (ap.isPresent("out")) { + ps = new PrintStream(new FileOutputStream(ap.getStringValue("out"))); + } + cc.run(ap.getStringValue("sites.file", null), + ap.getStringValue("tc.file", null), ap.getStringValue("config", null), System.out); + System.exit(0); + } + catch (Exception e) { + System.out.println("Failed to convert configuration: " + e.getMessage()); + e.printStackTrace(); + System.exit(2); + } + } + + private static void usage(ArgumentParser ap) { + System.out.println( + "Converts an existing set of sites.xml, tc.data, and swift.properties \n" + + "files (at least one of which needs to be specified) to a unified Swift\n" + + "configuration file.\n"); + ap.usage(); + } +} Added: trunk/src/org/griphyn/vdl/util/SwiftConfig.java =================================================================== --- trunk/src/org/griphyn/vdl/util/SwiftConfig.java (rev 0) +++ trunk/src/org/griphyn/vdl/util/SwiftConfig.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -0,0 +1,632 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 5, 2014 + */ +package org.griphyn.vdl.util; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.apache.log4j.Logger; +import org.globus.cog.abstraction.impl.common.AbstractionFactory; +import org.globus.cog.abstraction.impl.common.ProviderMethodException; +import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl; +import org.globus.cog.abstraction.impl.common.task.InvalidProviderException; +import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl; +import org.globus.cog.abstraction.impl.common.task.ServiceImpl; +import org.globus.cog.abstraction.interfaces.ExecutionService; +import org.globus.cog.abstraction.interfaces.Service; +import org.globus.cog.abstraction.interfaces.ServiceContact; +import org.globus.cog.karajan.util.BoundContact; +import org.globus.swift.catalog.site.Application; +import org.globus.swift.catalog.site.SwiftContact; +import org.globus.swift.catalog.site.SwiftContactSet; +import org.griphyn.vdl.util.ConfigTree.Node; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigIncludeContext; +import com.typesafe.config.ConfigIncluder; +import com.typesafe.config.ConfigObject; +import com.typesafe.config.ConfigParseOptions; +import com.typesafe.config.ConfigSyntax; +import com.typesafe.config.ConfigValue; +import com.typesafe.config.ConfigValueType; + +public class SwiftConfig implements Cloneable { + public static final Logger logger = Logger.getLogger(SwiftConfig.class); + + public static final boolean CHECK_DYNAMIC_NAMES = true; + public static final List DEFAULT_LOCATIONS; + + public enum Key { + DM_CHECKER("mappingCheckerEnabled"), + PROVENANCE_LOG("logProvenance"), + FILE_GC_ENABLED("fileGCEnabled"), + TICKER_ENABLED("tickerEnabled"), + TICKER_DATE_FORMAT("tickerDateFormat"), + TICKER_PREFIX("tickerPrefix"), + TRACING_ENABLED("tracingEnabled"), + FOREACH_MAX_THREADS("maxForeachThreads"), + CACHING_ALGORITHM("cachingAlgorithm"), + REPLICATION_ENABLED("replicationEnabled"), + WRAPPER_STAGING_LOCAL_SERVER("wrapperStagingLocalServer"), + REPLICATION_MIN_QUEUE_TIME("replicationMinQueueTime"), + REPLICATION_LIMIT("replicationLimit"), + WRAPPER_INVOCATION_MODE("wrapperInvocationMode"); + + public String propName; + private Key(String propName) { + this.propName = propName; + } + } + + public static SwiftConfigSchema SCHEMA; + + static { + SCHEMA = new SwiftConfigSchema(); + DEFAULT_LOCATIONS = new ArrayList(); + if (System.getenv("SWIFT_SITE_CONF") != null) { + DEFAULT_LOCATIONS.add(System.getenv("SWIFT_SITE_CONF")); + } + String swiftHome = System.getProperty("swift.home"); + if (swiftHome == null) { + swiftHome = System.getenv("SWIFT_HOME"); + if (swiftHome == null) { + throw new IllegalStateException("SWIFT_HOME is not set"); + } + } + + DEFAULT_LOCATIONS.add(swiftHome + File.separator + + "etc" + File.separator + "swift.conf"); + + // check keys against schema + + for (Key k : Key.values()) { + if (!SCHEMA.isNameValid(k.propName)) { + throw new IllegalArgumentException("Invalid property name for config key '" + k + "': " + k.propName); + } + } + } + + private static class KVPair { + public final String key; + public final String value; + + public KVPair(String key, String value) { + this.key = key; + this.value = value; + } + } + + private static class IncluderWrapper implements ConfigIncluder { + private final ConfigIncluder d; + + public IncluderWrapper(ConfigIncluder d) { + this.d = d; + } + + @Override + public ConfigIncluder withFallback(ConfigIncluder fallback) { + return this; + } + + @Override + public ConfigObject include(ConfigIncludeContext context, String what) { + int b = what.indexOf("${"); + while (b != -1) { + int e = what.indexOf("}", b); + String var = what.substring(b + 2, e); + what = what.substring(0, b) + resolve(var) + what.substring(e + 1); + b = what.indexOf("${"); + } + return ConfigFactory.parseFile(new File(what)).root(); + } + + private String resolve(String var) { + String v = null; + if (var.startsWith("env.")) { + v = System.getenv(var.substring(4)); + } + else { + v = System.getProperty(var); + } + if (v == null) { + throw new IllegalArgumentException("No such system property or environment variable: '" + var + "'"); + } + return v; + } + } + + public static SwiftConfig load(String fileName) { + return load(fileName, null); + } + + public static SwiftConfig load(String fileName, Map override) { + logger.info("Loading swift configuration file: " + fileName); + ConfigParseOptions opt = ConfigParseOptions.defaults(); + opt = opt.setIncluder(new IncluderWrapper(opt.getIncluder())). + setSyntax(ConfigSyntax.CONF).setAllowMissing(false); + Config conf = ConfigFactory.parseFile(new File(fileName), opt); + Config oconf = ConfigFactory.parseMap(override, ""); + conf = oconf.withFallback(conf); + conf = conf.resolveWith(getSubstitutions()); + ConfigTree out = SCHEMA.validate(conf); + SwiftConfig sc = new SwiftConfig(); + sc.setFileName(fileName); + sc.build(out); + return sc; + } + + private static Config getSubstitutions() { + Map m = new HashMap(); + + for (Map.Entry e : System.getenv().entrySet()) { + m.put("env." + e.getKey(), e.getValue()); + } + + return ConfigFactory.parseMap(m).withFallback(ConfigFactory.parseProperties(System.getProperties())); + } + + public static SwiftConfig load() { + for (String loc : DEFAULT_LOCATIONS) { + if (new File(loc).exists()) { + return load(loc); + } + } + throw new IllegalStateException("Could not find swift configuration file"); + } + + private static SwiftConfig _default; + + public synchronized static SwiftConfig getDefault() { + if (_default == null) { + _default = load(); + } + return _default; + } + + public synchronized static void setDefault(SwiftConfig conf) { + _default = conf; + } + + private SwiftContactSet definedSites; + private SwiftContactSet sites; + private ConfigTree tree; + private Map flat; + private String fileName; + + public SwiftConfig() { + definedSites = new SwiftContactSet(); + sites = new SwiftContactSet(); + flat = new HashMap(); + } + + public SwiftContactSet getSites() { + return sites; + } + + public SwiftContactSet getDefinedSites() { + return definedSites; + } + + public Collection getDefinedSiteNames() { + Set s = new TreeSet(); + for (BoundContact bc : definedSites.getContacts()) { + s.add(bc.getName()); + } + return s; + } + + public void setProperty(String key, Object value) { + flat.put(key, value); + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + @SuppressWarnings("unchecked") + private void build(ConfigTree tree) { + this.tree = tree; + List sites = null; + for (Map.Entry> e : tree.entrySet()) { + if (e.getKey().equals("site")) { + for (Map.Entry> f : e.getValue().entrySet()) { + site(definedSites, f.getKey(), f.getValue()); + } + } + else if (e.getKey().equals("sites")) { + sites = (List) getObject(e.getValue()); + } + else if (e.getKey().equals("app")) { + SwiftContact dummy = new SwiftContact(); + apps(dummy, e.getValue()); + for (Application app : dummy.getApplications()) { + definedSites.addApplication(app); + } + } + } + if (sites == null || sites.isEmpty()) { + throw new RuntimeException("No sites enabled"); + } + for (String siteName : sites) { + this.sites.addContact((SwiftContact) definedSites.getContact(siteName)); + } + this.sites.getApplications().putAll(definedSites.getApplications()); + + for (String leaf : tree.getLeafPaths()) { + flat.put(leaf, tree.get(leaf)); + } + } + + private void apps(SwiftContact sc, ConfigTree.Node n) { + /* + * app."*" { + * ... + * } + */ + + for (Map.Entry> e : n.entrySet()) { + String k = e.getKey(); + ConfigTree.Node c = e.getValue(); + + if (e.getKey().equals("ALL")) { + sc.addApplication(app("*", e.getValue())); + } + else { + sc.addApplication(app(removeQuotes(e.getKey()), e.getValue())); + } + } + + Application all = sc.getApplication("*"); + if (all != null) { + mergeEnvsToApps(sc, all.getEnv()); + mergePropsToApps(sc, all.getProperties()); + } + } + + private String removeQuotes(String key) { + if (key.startsWith("\"") && key.endsWith("\"")) { + return key.substring(1, key.length() - 2); + } + else { + return key; + } + } + + private void mergeEnvsToApps(SwiftContact bc, Map envs) { + for (Application app : bc.getApplications()) { + for (Map.Entry e : envs.entrySet()) { + if (!app.getEnv().containsKey(e.getKey())) { + // only merge if app does not override + app.setEnv(e.getKey(), e.getValue()); + } + } + } + } + + private void mergePropsToApps(SwiftContact bc, Map props) { + for (Application app : bc.getApplications()) { + for (Map.Entry e : props.entrySet()) { + if (!app.getProperties().containsKey(e.getKey())) { + app.addProperty(e.getKey(), e.getValue()); + } + } + } + } + + private Application app(String name, ConfigTree.Node n) { + /* + * app."*" { + * executable: "?String" + * + * options: "?Object" + * jobType: "?String", queue: "?String", project: "?String" + * maxWallTime: "?Time" + * + * env."*": "String" + * } + */ + + Application app = new Application(); + app.setName(name); + + for (Map.Entry> e : n.entrySet()) { + String k = e.getKey(); + ConfigTree.Node c = e.getValue(); + + if (k.equals("executable")) { + app.setExecutable(getString(c)); + } + else if (k.equals("options")) { + @SuppressWarnings("unchecked") + Map opt = (Map) getObject(c, "options"); + for (Map.Entry f : opt.entrySet()) { + app.addProperty(f.getKey(), f.getValue()); + } + } + else if (k.equals("jobQueue")) { + app.addProperty("queue", getString(c)); + } + else if (k.equals("jobProject")) { + app.addProperty("project", getString(c)); + } + else if (k.equals("env")) { + List envs = envs(c); + for (KVPair env : envs) { + app.setEnv(env.key, env.value); + } + } + else { + app.addProperty(k, getString(c)); + } + } + + return app; + } + + + private List envs(Node n) { + List l = new ArrayList(); + for (Map.Entry> e : n.entrySet()) { + l.add(new KVPair(e.getKey(), getString(e.getValue()))); + } + return l; + } + + private void site(SwiftContactSet sites, String name, ConfigTree.Node n) { + try { + SwiftContact sc = new SwiftContact(name); + + if (n.hasKey("OS")) { + sc.setProperty("sysinfo", getString(n, "OS")); + } + + + for (Map.Entry> e : n.entrySet()) { + String ctype = e.getKey(); + ConfigTree.Node c = e.getValue(); + + if (ctype.equals("execution")) { + sc.addService(execution(c)); + } + else if (ctype.equals("filesystem")) { + sc.addService(filesystem(c)); + } + else if (ctype.equals("workDirectory")) { + sc.setProperty("workdir", getString(c)); + } + else if (ctype.equals("scratch")) { + sc.setProperty("scratch", getString(c)); + } + else if (ctype.equals("app")) { + apps(sc, c); + } + else if (ctype.equals("options")) { + @SuppressWarnings("unchecked") + Map opt = (Map) getObject(c, "options"); + for (Map.Entry f : opt.entrySet()) { + sc.setProperty(f.getKey(), f.getValue()); + } + } + else { + sc.setProperty(ctype, getObject(c)); + } + } + sites.addContact(sc); + } + catch (Exception e) { + throw new RuntimeException("Invalid site entry '" + name + "': ", e); + } + } + + private Service filesystem(Node c) throws InvalidProviderException, ProviderMethodException { + Service s = new ServiceImpl(); + s.setType(Service.FILE_OPERATION); + service(c, s); + return s; + } + + private Service execution(ConfigTree.Node n) throws InvalidProviderException, ProviderMethodException { + ExecutionService s = new ExecutionServiceImpl(); + service(n, s); + return s; + } + + private void service(Node n, Service s) throws InvalidProviderException, ProviderMethodException { + String provider = null; + String url = null; + for (Map.Entry> e : n.entrySet()) { + String k = e.getKey(); + ConfigTree.Node c = e.getValue(); + + if (k.equals("type")) { + provider = getString(c); + } + else if (k.equals("URL")) { + url = getString(c); + } + else if (k.equals("jobManager")) { + ((ExecutionService) s).setJobManager(getString(c)); + } + else if (k.equals("jobProject")) { + s.setAttribute("project", getObject(c)); + } + else if (k.equals("maxJobs")) { + s.setAttribute("slots", getObject(c)); + } + else if (k.equals("maxJobTime")) { + s.setAttribute("maxTime", getObject(c)); + } + else if (k.equals("maxNodesPerJob")) { + s.setAttribute("maxNodes", getObject(c)); + } + else if (k.equals("jobQueue")) { + s.setAttribute("queue", getObject(c)); + } + else { + s.setAttribute(k, getObject(c)); + } + } + + s.setProvider(provider); + if (url != null) { + ServiceContact contact = new ServiceContactImpl(url); + s.setServiceContact(contact); + s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact)); + } + } + + private String getString(Node c) { + return (String) c.get(); + } + + private Object getObject(ConfigTree.Node c) { + return c.get(); + } + + private String getString(ConfigTree.Node c, String key) { + return (String) c.get(key); + } + + private Object getObject(ConfigTree.Node c, String key) { + return c.get(key); + } + + private void checkType(String name, ConfigValue value, ConfigValueType type) { + if (!type.equals(value.valueType())) { + throw new SwiftConfigException(value.origin(), + "'" + name + "': wrong type (" + value.valueType() + "). Must be a " + type); + } + } + + public Object clone() { + return this; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + SortedSet s = new TreeSet(flat.keySet()); + for (String k : s) { + sb.append(k); + sb.append(": "); + Object o = flat.get(k); + if (o instanceof String) { + sb.append('"'); + sb.append(o); + sb.append('"'); + } + else { + sb.append(o); + } + sb.append('\n'); + } + return sb.toString(); + } + + private void check(String name) { + if (CHECK_DYNAMIC_NAMES) { + if (!SCHEMA.isNameValid(name)) { + throw new IllegalArgumentException("Unknown property name: '" + name + "'"); + } + } + } + + private Object get(Key k) { + return flat.get(k.propName); + } + + public Object getProperty(String name) { + check(name); + return flat.get(name); + } + + public String getStringProperty(String name) { + check(name); + return (String) flat.get(name); + } + + public boolean isFileGCEnabled() { + return (Boolean) get(Key.FILE_GC_ENABLED); + } + + public boolean isTickerEnabled() { + return (Boolean) get(Key.TICKER_ENABLED); + } + + public String getTickerDateFormat() { + return (String) get(Key.TICKER_DATE_FORMAT); + } + + public String getTickerPrefix() { + return (String) get(Key.TICKER_PREFIX); + } + + public boolean isTracingEnabled() { + return (Boolean) get(Key.TRACING_ENABLED); + } + + public Object getProperty(String name, Object defVal) { + check(name); + Object v = flat.get(name); + if (v == null) { + return defVal; + } + else { + return v; + } + } + + public int getForeachMaxThreads() { + return (Integer) get(Key.FOREACH_MAX_THREADS); + } + + public String getCachingAlgorithm() { + return (String) get(Key.CACHING_ALGORITHM); + } + + public boolean isReplicationEnabled() { + return (Boolean) get(Key.REPLICATION_ENABLED); + } + + public String getWrapperStagingLocalServer() { + return (String) get(Key.WRAPPER_STAGING_LOCAL_SERVER); + } + + public boolean isProvenanceEnabled() { + return (Boolean) get(Key.PROVENANCE_LOG); + } + + public int getReplicationMinQueueTime() { + return (Integer) get(Key.REPLICATION_MIN_QUEUE_TIME); + } + + public int getReplicationLimit() { + return (Integer) get(Key.REPLICATION_LIMIT); + } + + public String getWrapperInvocationMode() { + return (String) get(Key.WRAPPER_INVOCATION_MODE); + } + + public boolean isMappingCheckerEnabled() { + return (Boolean) get(Key.DM_CHECKER); + } +} Added: trunk/src/org/griphyn/vdl/util/SwiftConfigException.java =================================================================== --- trunk/src/org/griphyn/vdl/util/SwiftConfigException.java (rev 0) +++ trunk/src/org/griphyn/vdl/util/SwiftConfigException.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -0,0 +1,20 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 5, 2014 + */ +package org.griphyn.vdl.util; + +import com.typesafe.config.ConfigOrigin; + +public class SwiftConfigException extends RuntimeException { + private ConfigOrigin loc; + + public SwiftConfigException(ConfigOrigin loc, String message) { + super((loc.filename() == null ? loc.description() : loc.filename() + ":" + loc.lineNumber()) + " " + message); + } +} Added: trunk/src/org/griphyn/vdl/util/SwiftConfigInfo.java =================================================================== --- trunk/src/org/griphyn/vdl/util/SwiftConfigInfo.java (rev 0) +++ trunk/src/org/griphyn/vdl/util/SwiftConfigInfo.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -0,0 +1,71 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 8, 2014 + */ +package org.griphyn.vdl.util; + +import java.util.Collection; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.globus.cog.util.ArgumentParser; +import org.globus.cog.util.StringUtil; + +public class SwiftConfigInfo { + public static void main(String[] args) { + ArgumentParser ap = new ArgumentParser(); + ap.setExecutableName("swift-config-info"); + ap.addFlag("help", "Prints usage information"); + ap.addFlag("list", "Lists all available configuration properties"); + ap.addOption(ArgumentParser.DEFAULT, "A property to print information about", + "property", ArgumentParser.OPTIONAL); + try { + ap.parse(args); + if (ap.isPresent("help")) { + ap.usage(); + System.exit(0); + } + if (ap.isPresent("list")) { + list(); + } + if (ap.hasValue(ArgumentParser.DEFAULT)) { + info(ap.getStringValue(ArgumentParser.DEFAULT)); + } + System.exit(0); + } + catch (Exception e) { + System.out.println("Error parsing command line: " + e.getMessage()); + ap.usage(); + System.exit(1); + } + } + + private static void info(String prop) { + SwiftConfigSchema.Info info = SwiftConfig.SCHEMA.getPropertyDescriptions().get(prop); + if (info != null) { + if (info.doc == null) { + System.out.println(prop + ": no description available"); + } + else { + System.out.println(prop + ":"); + System.out.println(StringUtil.wordWrap(info.doc, 6, 65)); + } + } + else { + System.out.println(prop + ": unknown property"); + } + } + + private static void list() { + Collection names = SwiftConfig.SCHEMA.listProperties(); + SortedSet snames = new TreeSet(names); + for (String name : snames) { + System.out.println(name.replace("\"*\"", "*")); + } + } +} Added: trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java =================================================================== --- trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java (rev 0) +++ trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -0,0 +1,288 @@ +//---------------------------------------------------------------------- +//This code is developed as part of the Java CoG Kit project +//The terms of the license can be found at http://www.cogkit.org/license +//This message may not be removed or altered. +//---------------------------------------------------------------------- + +/* + * Created on Jul 5, 2014 + */ +package org.griphyn.vdl.util; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import com.typesafe.config.ConfigOrigin; +import com.typesafe.config.ConfigValue; + +public class SwiftConfigSchema { + private static final String STAR = "\"*\""; + + public static class Info { + public ConfigPropertyType type; + public Object value; + public boolean optional; + public String doc; + public ConfigOrigin loc; + } + + private Config schema; + private ConfigTree info; + private Map descriptions; + private Set validNames; + + public SwiftConfigSchema() { + validNames = new HashSet(); + schema = ConfigFactory.parseResources("swift.conf.schema"); + schema = schema.resolve(); + if (schema.isEmpty()) { + throw new RuntimeException("Could not find swift.conf.schema"); + } + info = new ConfigTree(); + for (Map.Entry e : schema.entrySet()) { + String k = e.getKey(); + String type = null; + Object defaultValue = null; + String doc = null; + ConfigOrigin loc = null; + if (k.endsWith(".\"_type\"")) { + type = schema.getString(k); + k = k.substring(0, k.lastIndexOf('.')); + loc = e.getValue().origin(); + } + else if (k.indexOf(".\"_") == -1){ + type = schema.getString(k); + loc = e.getValue().origin(); + } + else if (k.endsWith(".\"_default\"")){ + defaultValue = e.getValue().unwrapped(); + k = k.substring(0, k.lastIndexOf('.')); + } + else if (k.endsWith(".\"_doc\"")){ + doc = stripDoc((String) e.getValue().unwrapped()); + k = k.substring(0, k.lastIndexOf('.')); + } + else if (k.indexOf(".\"_") != -1) { + continue; + } + Info i = info.get(k); + if (i == null) { + i = new Info(); + info.put(k, i); + validNames.add(k); + } + if (type != null) { + if (type.startsWith("?")) { + i.optional = true; + type = type.substring(1); + } + i.type = getTypeInstance(type, e.getValue()); + } + if (defaultValue != null) { + i.value = defaultValue; + } + if (doc != null) { + i.doc = doc; + } + if (loc != null) { + i.loc = loc; + } + } + } + + private String stripDoc(String doc) { + return doc.replaceAll("[\\t\\n]+", ""); + } + + private ConfigPropertyType getTypeInstance(String type, ConfigValue value) { + if (type.equals("String")) { + return ConfigPropertyType.STRING; + } + else if (type.equals("Boolean")) { + return ConfigPropertyType.BOOLEAN; + } + else if (type.equals("Int")) { + return ConfigPropertyType.INT; + } + else if (type.equals("Float")) { + return ConfigPropertyType.FLOAT; + } + else if (type.equals("StrictlyPositiveInt")) { + return ConfigPropertyType.STRICTLY_POSITIVE_INT; + } + else if (type.equals("Throttle")) { + return ConfigPropertyType.THROTTLE; + } + else if (type.equals("PortRange")) { + return ConfigPropertyType.PORT_RANGE; + } + else if (type.equals("PositiveInt")) { + return ConfigPropertyType.POSITIVE_INT; + } + else if (type.equals("PositiveFloat")) { + return ConfigPropertyType.POSITIVE_FLOAT; + } + else if (type.equals("Time")) { + return ConfigPropertyType.TIME; + } + else if (type.equals("Seconds")) { + return ConfigPropertyType.STRICTLY_POSITIVE_INT; + } + else if (type.equals("OS")) { + return ConfigPropertyType.OS; + } + else if (type.equals("StringList")) { + return ConfigPropertyType.STRING_LIST; + } + else if (type.equals("Object")) { + return ConfigPropertyType.OBJECT; + } + else if (type.startsWith("Choice[")) { + String values = type.substring("Choice[".length(), type.length() - 1); + return new ConfigPropertyType.Choices(values.split(",\\s*")); + } + else if (type.startsWith("Interval[")) { + String values = type.substring("Interval[".length(), type.length() - 1); + String[] s = values.split(",\\s*", 2); + return new ConfigPropertyType.Interval(Double.parseDouble(s[0]), Double.parseDouble(s[1])); + } + else { + throw new IllegalArgumentException(loc(value.origin()) + ": unknown property type: '" + type + "'"); + } + } + + private String loc(ConfigOrigin o) { + return o.filename() + ":" + o.lineNumber(); + } + + public ConfigTree validate(Config conf) { + ConfigTree validated = new ConfigTree(); + + // build a tree of the actual config so we can easily check them for missing properties + ConfigTree confTree = new ConfigTree(); + + // check things in config against schema + for (Map.Entry e : conf.entrySet()) { + String k = e.getKey(); + confTree.put(k, Boolean.TRUE); + // check if the properties are defined in the schema + Info i = info.get(k, STAR); + if (i == null) { + i = info.get(k, STAR); + throw new SwiftConfigException(e.getValue().origin(), "unexpected property '" + k + "'"); + } + // now check values + if (i.type == null) { + throw new IllegalStateException("Missing type for key " + k); + } + Object value = checkValue(k, e.getValue(), i.type); + validated.put(k, value); + } + + // check for missing things + for (String key : info.getLeafPaths()) { + Info i = info.get(key); + String notFound = findMissing(key, confTree, i, validated); + if (notFound != null) { + findMissing(key, confTree, i, validated); + ConfigOrigin loc = info.get(key).loc; + throw new SwiftConfigException(conf.origin(), "missing property '" + notFound + + "' defined in " + loc.filename() + ":" + loc.lineNumber()); + } + } + return validated; + } + + private String findMissing(String key, ConfigTree confTree, Info i, ConfigTree validated) { + List found = confTree.expandWildcards(key, STAR); + for (String f : found) { + if (!confTree.hasKey(f)) { + if (i.optional) { + if (i.value != null) { + validated.put(f, i.value); + } + } + else { + return f; + } + } + } + return null; + } + + private void setValue(Map validated, Config conf, String k) { + if (!conf.hasPath(k)) { + String defKey = k + ".\"_default\""; + if (schema.hasPath(defKey)) { + validated.put(k, schema.getValue(defKey).unwrapped()); + } + } + else { + validated.put(k, conf.getValue(k).unwrapped()); + } + } + + private Object checkValue(String k, ConfigValue value, ConfigPropertyType t) { + Object v = value.unwrapped(); + switch (value.valueType()) { + case STRING: + if (t.getBaseType() != ConfigPropertyType.STRING && t.getBaseType() != ConfigPropertyType.OBJECT) { + throw invalidValue(value, k, v, t.getBaseType()); + } + return t.check(k, value.unwrapped(), null); + case NUMBER: + if (t.getBaseType() != ConfigPropertyType.INT && t.getBaseType() != ConfigPropertyType.FLOAT) { + throw invalidValue(value, k, v, t.getBaseType()); + } + if (t.getBaseType() == ConfigPropertyType.INT) { + Number n = (Number) value.unwrapped(); + if (n.intValue() != n.doubleValue()) { + throw invalidValue(value, k, v, t.getBaseType()); + } + } + return t.check(k, v, null); + case BOOLEAN: + if (t.getBaseType() != ConfigPropertyType.BOOLEAN) { + throw invalidValue(value, k, v, t.getBaseType()); + } + return value.unwrapped(); + default: + return t.check(k, v, null); + } + } + + private SwiftConfigException invalidValue(ConfigValue value, String k, Object v, ConfigPropertyType t) { + return new SwiftConfigException(value.origin(), "invalid value '" + v + "' for property '" + k + "'. Expected a " + t); + } + + public synchronized Map getPropertyDescriptions() { + if (descriptions == null) { + descriptions = new HashMap(); + for (String s : info.getLeafPaths()) { + Info i = info.get(s); + if (i != null && i.doc != null) { + descriptions.put(s, i); + } + } + } + return descriptions; + } + + public boolean propertyExists(String name) { + return info.get(name) != null; + } + + public boolean isNameValid(String name) { + return validNames.contains(name); + } + + public Collection listProperties() { + return validNames; + } +} Deleted: trunk/src/org/griphyn/vdl/util/TriStateBoolean.java =================================================================== --- trunk/src/org/griphyn/vdl/util/TriStateBoolean.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/util/TriStateBoolean.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -1,66 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Feb 7, 2007 - */ -package org.griphyn.vdl.util; - -public final class TriStateBoolean { - public static final TriStateBoolean FALSE = new TriStateBoolean(0); - public static final TriStateBoolean TRUE = new TriStateBoolean(1); - public static final TriStateBoolean MAYBE = new TriStateBoolean(2); - - private int value; - - private TriStateBoolean(int value) { - this.value = value; - } - - public boolean equals(Object obj) { - if (obj instanceof TriStateBoolean) { - return value == ((TriStateBoolean) obj).value; - } - else { - return false; - } - } - - public int hashCode() { - return value; - } - - public String toString() { - switch(value) { - case 0: return "false"; - case 1: return "true"; - default: return "maybe"; - } - } - - public static TriStateBoolean valueOf(String value) { - if ("false".equalsIgnoreCase(value) || "no".equalsIgnoreCase(value)) { - return FALSE; - } - else if ("true".equalsIgnoreCase(value) || "yes".equalsIgnoreCase(value)) { - return TRUE; - } - else { - return MAYBE; - } - } -} Deleted: trunk/src/org/griphyn/vdl/util/VDL2Config.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -1,316 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Dec 5, 2006 - */ -package org.griphyn.vdl.util; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.log4j.Logger; -import org.globus.cog.karajan.util.BoundContact; -import org.globus.common.CoGProperties; - -public class VDL2Config extends Properties { - - private static final long serialVersionUID = 1L; - - public static final Logger logger = Logger.getLogger(VDL2Config.class); - - public static final String CONFIG_FILE_NAME = "swift.properties"; - public static final String[] CONFIG_FILE_SEARCH_PATH = new String[] { - System.getProperty("swift.home") + File.separator + "etc" + File.separator - + CONFIG_FILE_NAME, - System.getProperty("user.home") + File.separator + ".swift" + File.separator - + CONFIG_FILE_NAME, - System.getProperty("vds.home") + File.separator + "etc" + File.separator - + CONFIG_FILE_NAME }; - - private static VDL2Config config; - - public static VDL2Config getConfig() throws IOException { - if(config == null) { - config = getDefaultConfig(); - } - return config.check(); - } - - public static synchronized VDL2Config getDefaultConfig() throws IOException { - if (config == null) { - config = new VDL2Config(); - for (int i = 0; i < CONFIG_FILE_SEARCH_PATH.length; i++) { - config.load(CONFIG_FILE_SEARCH_PATH[i]); - } - } - return config; - } - - public static VDL2Config getConfig(String file) throws IOException { - VDL2Config c; - try { - VDL2Config d = getConfig(); - c = new VDL2Config(d); - } - catch (Exception e) { - c = new VDL2Config(); - } - c.load(file); - config = c; - config.check(); - return config; - } - - private List files, tried; - private Map types; - private Map propertySource; - private String currentFile; - - private VDL2Config() { - files = new LinkedList(); - tried = new LinkedList(); - propertySource = new HashMap(); - types = new HashMap(); - put(VDL2ConfigProperties.POOL_FILE, "${swift.home}/etc/sites.xml", ConfigPropertyType.FILE); - put(VDL2ConfigProperties.TC_FILE, "${swift.home}/etc/tc.data", ConfigPropertyType.FILE); - put(VDL2ConfigProperties.LAZY_ERRORS, "false", ConfigPropertyType.BOOLEAN); - put(VDL2ConfigProperties.CACHING_ALGORITHM, "LRU", ConfigPropertyType.STRING); - put(VDL2ConfigProperties.CLUSTERING_ENABLED, "false", ConfigPropertyType.BOOLEAN); - put(VDL2ConfigProperties.CLUSTERING_QUEUE_DELAY, "4", ConfigPropertyType.INT); - put(VDL2ConfigProperties.CLUSTERING_MIN_TIME, "60", ConfigPropertyType.INT); - put("throttle.submit", "4", ConfigPropertyType.INT); - put("throttle.host.submit", "2", ConfigPropertyType.INT); - put("throttle.transfers", "4", ConfigPropertyType.INT); - put("throttle.file.operations", "8", ConfigPropertyType.INT); - put("throttle.score.job.factor", "4", ConfigPropertyType.FLOAT); - put(VDL2ConfigProperties.SITEDIR_KEEP, "false", ConfigPropertyType.BOOLEAN); - put(VDL2ConfigProperties.PROVENANCE_LOG, "false", ConfigPropertyType.BOOLEAN); - - put("execution.retries", "0", ConfigPropertyType.INT); - - put("replication.enabled", "false", ConfigPropertyType.BOOLEAN); - put("replication.min.queue.time", "60", ConfigPropertyType.INT); - put("replication.limit", "3", ConfigPropertyType.INT); - put("status.mode", "files", ConfigPropertyType.choices("files", "provider")); - put("wrapper.parameter.mode", "args", ConfigPropertyType.choices("args", "files")); - put("wrapper.invocation.mode", "absolute", ConfigPropertyType.choices("absolute", "relative")); - - // TODO what are the valid values here? - put("cdm.broadcast.mode", "file"); - put("use.provider.staging", "false", ConfigPropertyType.BOOLEAN); - put("use.wrapper.staging", "false", ConfigPropertyType.BOOLEAN); - put("ticker.date.format", "", ConfigPropertyType.STRING); - put("ticker.prefix", "Progress: ", ConfigPropertyType.STRING); - - put(VDL2ConfigProperties.FILE_GC_ENABLED, "true", ConfigPropertyType.BOOLEAN); - put(VDL2ConfigProperties.DM_CHECKER, "on", ConfigPropertyType.ONOFF); - } - - private VDL2Config(VDL2Config other) { - this.putAll(other); - this.files.addAll(other.files); - this.propertySource.putAll(other.propertySource); - } - - protected void load(String file) throws IOException { - this.currentFile = file; - tried.add(file); - File f = new File(file); - if (f.exists()) { - files.add(file); - super.load(new FileInputStream(f)); - } - String hostname = getHostName(); - if (hostname != null) { - CoGProperties.getDefault().setHostName(hostname); - } - String ip = getIP(); - if (ip != null) { - CoGProperties.getDefault().setIPAddress(ip); - } - String tcpPortRange = getTCPPortRange(); - if (tcpPortRange != null) { - CoGProperties.getDefault().put("tcp.port.range", tcpPortRange); - } - } - - public void validateProperties() { - for (Map.Entry e : this.entrySet()) { - checkType(e.getKey(), e.getValue()); - } - } - - protected VDL2Config check() throws IOException { - if (files.size() == 0) { - throw new FileNotFoundException("No Swift configuration file found. Tried " + tried); - } - else { - return this; - } - } - - public synchronized Object put(Object key, Object value, ConfigPropertyType type) { - types.put(key, type); - return put(key, value); - } - - /** - * Overridden to do variable expansion. Variables will be expanded if there - * is a system property with that name. Otherwise, the expansion will not - * occur. - */ - public synchronized Object put(Object key, Object value) { - propertySource.put(key, currentFile); - String svalue = (String) value; - if (svalue.indexOf("${") == -1) { - return super.put(key, value); - } - else { - StringBuffer sb = new StringBuffer(); - int index = 0, last = 0; - while (index >= 0) { - index = svalue.indexOf("${", index); - if (index >= 0) { - if (last != index) { - sb.append(svalue.substring(last, index)); - last = index; - } - int end = svalue.indexOf("}", index); - if (end == -1) { - sb.append(svalue.substring(index)); - break; - } - else { - String name = svalue.substring(index + 2, end); - String pval = System.getProperty(name); - index = end + 1; - if (pval == null) { - continue; - } - else { - sb.append(pval); - last = index; - } - } - } - } - sb.append(svalue.substring(last)); - value = sb.toString(); - return super.put(key, value); - } - } - - private void checkType(Object key, Object value) { - ConfigPropertyType type = types.get(key); - if (type != null) { - type.checkValue((String) key, (String) value, propertySource.get(key)); - } - } - - public String getPoolFile() { - return getProperty(VDL2ConfigProperties.POOL_FILE); - } - - public String getTCFile() { - return getProperty(VDL2ConfigProperties.TC_FILE); - } - - public String getIP() { - return getProperty(VDL2ConfigProperties.IP_ADDRESS); - } - - public String getHostName() { - return getProperty(VDL2ConfigProperties.HOST_NAME); - } - - public String getTCPPortRange() { - return getProperty(VDL2ConfigProperties.TCP_PORT_RANGE); - } - - public boolean getLazyErrors() { - return Boolean.valueOf(getProperty(VDL2ConfigProperties.LAZY_ERRORS, "true")).booleanValue(); - } - - public TriStateBoolean getKickstartEnabled() { - return TriStateBoolean.valueOf(getProperty(VDL2ConfigProperties.KICKSTART_ENABLED, "false")); - } - - public boolean getSitedirKeep() { - return Boolean.valueOf(getProperty(VDL2ConfigProperties.SITEDIR_KEEP, "true")).booleanValue(); - } - - private Boolean provenanceLogCached; - - public boolean getProvenanceLog() { - if (provenanceLogCached == null) { - provenanceLogCached = Boolean.valueOf(getProperty(VDL2ConfigProperties.PROVENANCE_LOG, "false")); - } - return provenanceLogCached; - } - - public boolean isTracingEnabled() { - return Boolean.valueOf(getProperty(VDL2ConfigProperties.TRACING_ENABLED, "false")).booleanValue(); - } - - public String getTickerDateFormat() { - return getProperty("ticker.date.format"); - } - - public String getTickerPrefix() { - return getProperty("ticker.prefix"); - } - - public String toString() { - return "Swift configuration (" + files + "): " + super.toString(); - } - - public Object clone() { - VDL2Config conf = new VDL2Config(); - conf.putAll(this); - conf.files.addAll(files); - conf.propertySource.putAll(propertySource); - return conf; - } - - public String getProperty(String name, BoundContact bc) { - if(bc!=null) { - if(logger.isDebugEnabled()) { - logger.debug("Checking BoundContact "+bc+" for property "+name); - } - String prop = (String) bc.getProperty(name); - if(prop != null) { - return prop; - } - } - if(logger.isDebugEnabled()) { - logger.debug("Getting property "+name+" from global configuration"); - } - return getProperty(name); - } - - public void setCurrentFile(String f) { - this.currentFile = f; - } -} Deleted: trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java =================================================================== --- trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2014-07-09 19:38:06 UTC (rev 7975) +++ trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2014-07-09 19:38:50 UTC (rev 7976) @@ -1,171 +0,0 @@ -/* - * Copyright 2012 University of Chicago - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Created on Dec 23, 2006 - */ -package org.griphyn.vdl.util; - -import java.util.Map; -import java.util.TreeMap; - -public class VDL2ConfigProperties { - public static final String POOL_FILE = "sites.file"; - public static final String TC_FILE = "tc.file"; - public static final String IP_ADDRESS = "ip.address"; - public static final String HOST_NAME = "hostname"; - public static final String TCP_PORT_RANGE = "tcp.port.range"; - public static final String LAZY_ERRORS = "lazy.errors"; - public static final String PGRAPH = "pgraph"; - public static final String PGRAPH_GRAPH_OPTIONS = "pgraph.graph.options"; - public static final String PGRAPH_NODE_OPTIONS = "pgraph.node.options"; - public static final String CACHING_ALGORITHM = "caching.algorithm"; - public static final String CLUSTERING_ENABLED = "clustering.enabled"; - public static final String CLUSTERING_QUEUE_DELAY = "clustering.queue.delay"; - public static final String CLUSTERING_MIN_TIME = "clustering.min.time"; - public static final String KICKSTART_ENABLED = "kickstart.enabled"; - public static final String KICKSTART_ALWAYS_TRANSFER = "kickstart.always.transfer"; - public static final String WRAPPERLOG_ALWAYS_TRANSFER = "wrapperlog.always.transfer"; - public static final String SITEDIR_KEEP = "sitedir.keep"; - public static final String PROVENANCE_LOG = "provenance.log"; - public static final String FILE_GC_ENABLED = "file.gc.enabled"; - public static final String TRACING_ENABLED = "tracing.enabled"; - public static final String DM_CHECKER = "mapping.checker"; - public static final Map PROPERTIES; - - static { - PROPERTIES = new TreeMap(); - PROPERTIES.put(POOL_FILE, new PropInfo("file", - "Points to the location of the sites.xml file")); - PROPERTIES.put(TC_FILE, new PropInfo("file", "Points to the location of the tc.data file")); - PROPERTIES.put(IP_ADDRESS, new PropInfo("aaa.bbb.ccc.ddd", - "Can be used to specify a publicly reacheable IP address for " - + "this machine which is generally used for Globus callbacks. " - + "Normally this should be auto-detected, but if you have " - + "multiple network cards or NAT then you may need to set this")); - PROPERTIES.put(HOST_NAME, new PropInfo("string", - "Can be used to specify a publicly reacheable DNS name or IP address for " - + "this machine which is generally used for Globus callbacks. " - + "Normally this should be auto-detected, but if you do " - + "not have a public DNS name, you may want to set this.")); - PROPERTIES.put(TCP_PORT_RANGE, new PropInfo("start,end", - "A TCP port range can be specified to " - + "restrict the ports on which GRAM callback services are started. " - + "This is likely needed if your submit host is behind a firewall, " - + "in which case the firewall should be configured to allow " - + "incoming connections on ports in the range.")); - PROPERTIES.put(LAZY_ERRORS, new PropInfo("true|false", - "Use a lazy mode to deal with errors. When set to 'true' swift will proceed with the " - + "execution until no more data can be derived because of " - + "errors in dependent steps. If set to 'false', an error will " - + "cause the execution to immediately stop")); - PROPERTIES.put(PGRAPH, new PropInfo("true|false|", - "Whether to generate a provenance " - + "graph or not. If 'true' is used, the file name for the graph will " - + "be chosen by swift.")); - PROPERTIES.put(PGRAPH_GRAPH_OPTIONS, new PropInfo("", - "Graph options to be passed to the .dot file. " - + "These will appear in the 'graph' clause in the .dot file: " - + "graph []; ")); - PROPERTIES.put(PGRAPH_NODE_OPTIONS, new PropInfo("", - "Node options to be passed to the .dot file." - + "These will appear in the 'node' clause in the .dot file: " - + "node []; ")); - PROPERTIES.put(CACHING_ALGORITHM, new PropInfo("[LRU]", "The algorithm to use for the " - + "swift file caching mechanism. LRU is the only one available now.")); - PROPERTIES.put(CLUSTERING_ENABLED, new PropInfo("true|false", - "Whether to enable clustering of small jobs. If enabled, jobs with a " - + "max wall time which is less than the value of the " - + CLUSTERING_MIN_TIME - + " property will be clustered into one job which has a cummulative" - + " max wall time greater or equal to the value of the " - + CLUSTERING_MIN_TIME + " property.")); - PROPERTIES.put(CLUSTERING_QUEUE_DELAY, new PropInfo("", "The delay at which " - + "the clustering code scans the clustering queue. A job marked for clustering " - + "will spend no more than the value of this property in the clustering queue.")); - PROPERTIES.put(CLUSTERING_MIN_TIME, new PropInfo("", "The threshold determines " - + " if a job as being clusterable. Also represents the minimum cummulative " - + "wall time that a cluster will have.")); - PROPERTIES.put(KICKSTART_ENABLED, new PropInfo("", - "Controls the use of Kickstart by Swift. The \"maybe\" " - + "value tells Swift to use Kickstart on sites where it is available.")); - PROPERTIES.put( - KICKSTART_ALWAYS_TRANSFER, - new PropInfo( - "", - "If Kickstart is used (see \"" - + KICKSTART_ENABLED - + "\"), it controls when " - + "Kickstart records are transfered back to the submit host. If set to \"false\" " - + "Swift will only transfer a Kickstart record for a job when the job fails. " - + "If set to \"true\", Swift will transfer Kickstart records whether a job " - + "fails or not.")); - PROPERTIES.put( - WRAPPERLOG_ALWAYS_TRANSFER, - new PropInfo( - "", - "Controls when " - + "wrapper logs are transfered back to the submit host. If set to \"false\" " - + "Swift will only transfer a wrapper log for a job when the job fails. " - + "If set to \"true\", Swift will transfer wrapper logs whether a job " - + "fails or not.")); - PROPERTIES.put( - SITEDIR_KEEP, - new PropInfo( - "", - "If set to true, keeps remote site run directory after execution has completed.")); - - - PROPERTIES.put( - PROVENANCE_LOG, - new PropInfo( - "", - "If set to true, will record provenance information in the log file")); - - PROPERTIES.put(FILE_GC_ENABLED, new PropInfo("", "Allows disabling the file garbage collector. " + - "If set to false, files mapped by collectable mappers (such as the concurrent mapper) will not be " + - "deleted when their swift variables go out of scope.")); - - PROPERTIES.put(TRACING_ENABLED, new PropInfo("", "Enables execution tracing. If set to 'true', " + - "operations within swift such as iterations, invocations, assignments, and declarations, as well " + - "as data dependencies will be logged. This comes at a cost in performance. It is therefore " + - "disabled by default.")); - - PROPERTIES.put(DM_CHECKER, new PropInfo("", "Controls the run-time duplicate mapping checker " + - "(which indetifies mapping conflicts). If set to 'on', the checker is enabled. When enabled, a record " + - "of all mapped data is kept, so this comes at the expense of a slight memory leak. If set 'off', the" + - " mapping checker is disabled. Enabled by default.")); - } - - public static Map getPropertyDescriptions() { - return PROPERTIES; - } - - public static String getPropertyDescription(String name) { - return PROPERTIES.get(name).desc; - } - - public static class PropInfo { - public final String validValues; - public final String desc; - - public PropInfo(String validValues, String desc) { - this.validValues = validValues; - this.desc = desc; - } - } -} From hategan at ci.uchicago.edu Wed Jul 9 14:52:10 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 9 Jul 2014 14:52:10 -0500 (CDT) Subject: [Swift-commit] r7978 - trunk/bin Message-ID: <20140709195210.9D3A89D66E@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-09 14:52:10 -0500 (Wed, 09 Jul 2014) New Revision: 7978 Removed: trunk/bin/checksites.k Log: removed checksites.k tool; was not updated in ages Deleted: trunk/bin/checksites.k =================================================================== --- trunk/bin/checksites.k 2014-07-09 19:40:34 UTC (rev 7977) +++ trunk/bin/checksites.k 2014-07-09 19:52:10 UTC (rev 7978) @@ -1,75 +0,0 @@ -import("sys.k") -import("task.k") - - -element(pool, [handle, optional(gridlaunch), sysinfo, optional(lrc), ..., channel(properties)] - list(handle, each(...)) -) - -element(lrc, [url] -) - -element(gridftp, [url, storage, major, minor, patch] - if( - url == "local://localhost" - list("file", "local", "") - list("file", "gsiftp", url) - ) -) - -element(jobmanager, [universe, url, major, minor, patch] - provider := if( - major == "4" "GT4" - major == "2" "GT2" - url == "local://localhost" "local" - throw("Unknown job manager version: major = {major}, minor = {minor}, patch = {patch}") - ) - list("execution", provider, url) -) - -element(profile, [namespace, key, value] -) - -element(workdirectory, [dir] -) - -element(forkify, [url] - concat(first(split(url, "/")), "/jobmanager-fork") -) - -sitesFile := try(first(...), throw("Usage cog-workflow checksites.k ")) - -sites := list(executeFile(sitesFile)) - -for(site, sites - - name := first(site) - services := butFirst(site) - - print("Checking {name}... ") - - for(service, services - [type, provider, url] := each(service) - - print(" {provider}/{type}: ", nl=false) - - race( - try( - sequential( - if( - type == "file" discard(file:exists("blabla", host=url, provider=provider)) - type == "execution" execute("/bin/date", provider=provider, host=forkify(url)) - ) - print("[OK]") - ) - print(exception) - ) - sequential( - wait(delay=10000) - print("[timeout]") - ) - ) - ) -) - -print("All done") \ No newline at end of file From hategan at ci.uchicago.edu Wed Jul 9 15:00:23 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 9 Jul 2014 15:00:23 -0500 (CDT) Subject: [Swift-commit] r7979 - trunk/resources Message-ID: <20140709200023.692CC9D66E@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-09 15:00:23 -0500 (Wed, 09 Jul 2014) New Revision: 7979 Added: trunk/resources/swift.conf.schema Log: added config schema Added: trunk/resources/swift.conf.schema =================================================================== --- trunk/resources/swift.conf.schema (rev 0) +++ trunk/resources/swift.conf.schema 2014-07-09 20:00:23 UTC (rev 7979) @@ -0,0 +1,323 @@ + +app."*" { + _doc: + """ Defines an application global to all sites. The '*' symbol stands for + the name of the application, as would appear in a swift script. The string + ALL could be used to provide settings that apply to all applications.""" + + executable { + _type: "?String" + _doc: + """ Specifies the executable of an application. Can either be an absolute + path or and executable name that must be in the system PATH on the target + site or the symbol "*", in which case Swift will attempt to run an + executable with the same name as the application""" + } + + options { + _type: "?Object" + _doc: + """ Allows the definition of arbitrary application options. The meaning + of these options is largely dependent on the mechanism used to execute + the application. """ + } + + jobType: "?String", jobQueue: "?String", jobProject: "?String" + maxWallTime: "?Time" + + env."*": "String" +} + + +site."*" { + OS: "?OS" + maxParallelTasks { + _type: "?StrictlyPositiveInt" + _default: 2 + } + + initialParallelTasks { + _type: "?StrictlyPositiveInt" + _default: 2 + } + + delayBase { + _type: "?Float" + } + + maxSubmitRate { + _type: "?PositiveFloat" + } + + staging { + _type: "?Choice[swift, provider, wrapper]" + _default: "swift" + } + + stagingMethod { + _type: "?String" + _default: "proxy" + } + + filesystem { + type: "String" + URL { + _type: "?String" + _default: null + } + } + + execution { + type: "String" + URL: "?String" + jobManager: "?String" + # Coasters + maxJobs: "?StrictlyPositiveInt" + nodeGranularity: "?StrictlyPositiveInt" + tasksPerNode: "?StrictlyPositiveInt" + allocationStepSize: "?Interval[0.0, 1.0]" + lowOverallocation: "?StrictlyPositiveInt" + highOverallocation: "?StrictlyPositiveInt" + overallocationDecayFactor: "?PositiveFloat" + spread: "?Interval[0.0, 1.0]" + exponentialSpread: "?Int" + reserve: "?Seconds" + maxNodesPerJob: "?StrictlyPositiveInt" + maxJobTime: "?Time" + userHomeOverride: "?String" + internalHostName: "?String" + jobQueue: "?String" + jobProject: "?String" + workerLoggingLevel: "?Choice[ERROR, WARN, INFO, DEBUG, TRACE]" + workerLoggingDirectory: "?String" + softImage: "?String" + options: "?Object" + } + + workDirectory: "String" + app."*": { + executable: "?String" + + options: "?Object" + jobType: "?String", jobQueue: "?String", jobProject: "?String" + maxWallTime: "?Time" + + env."*": "String" + } + + wrapperInterpreter: "?String" + wrapperInterpreterOptions: "?StringList" + wrapperScript: "?String" + cleanupCommand: "?String" + cleanupCommandOptions: "?StringList" + + # Global properties that can also be used for individual sites + wrapperParameterMode: "?Choice[args, files]" + keepSiteDir: "?Boolean" + statusMode: "?Choice[files, provider]" + +} + +sites { + _type: "StringList" + _doc: + """Selects a list of sites to run on. The sites in question must have been defined + using the site.* properties. To get a list of sites defined in the current + configuration, start swift with the -sitelist command line option""" +} + +staging { + _type: "?Choice[swift, provider, wrapper]" + _default: "swift" +} + +lazyErrors { + _type: "?Boolean" + _default: false + _doc: + """Use a lazy mode to deal with errors. When set to 'true' Swift will proceed with the + execution until no more data can be derived because of errors in dependent steps. If + set to 'false', an error will cause the execution to immediately stop""" +} + +executionRetries { + _type: "?PositiveInt" + _default: 0 +} + +cachingAlgorithm { + _type: "?Choice[LRU]" + _default: "LRU" +} + +jobSubmitThrottle { + _type: "?Throttle" + _default: 4 +} + +hostJobSubmitThrottle { + _type: "?Throttle" + _default: 2 +} + +fileTransfersThrottle { + _type: "?Throttle" + _default: 4 +} + +fileOperationsThrottle { + _type: "?Throttle" + _default: 8 +} + +siteScoreThrottlingFactor { + _type: "?PositiveFloat" + _default: 0.2 +} + +keepSiteDir { + _type: "?Boolean" + _default: false + _doc: + """If set to true, it prevents the removal of remote site run directories + after execution has completed.""" +} + +logProvenance { + _type: "?Boolean" + _default: false + _doc: + """If set to true, will record provenance information in the log file.""" +} + +replicationEnabled { + _type: "?Boolean" + _default: false +} + +replicationMinQueueTime { + _type: "?StrictlyPositiveInt" + _default: 60 +} + +replicationLimit { + _type: "?StrictlyPositiveInt" + _default: 3 +} + +statusMode { + _type: "?Choice[files, provider]" + _default: "files" +} + +wrapperParameterMode { + _type: "?Choice[args, files]" + _default: "args" +} + +wrapperInvocationMode { + _type: "?Choice[absolute, relative]" + _default: "absolute" +} + +CDMBroadcastMode { + _type: "?String" + _default: "file" +} + +providerStagingPinSwiftFiles { + _type: "?Boolean" + _default: false +} + +tickerDateFormat { + _type: "?String" + _default: "" +} + +tickerPrefix { + _type: "?String" + _default: "Progress: " +} + +tickerEnabled { + _type: "?Boolean" + _default: true +} + +fileGCEnabled { + _type: "?Boolean" + _default: true + _doc: + """Allows disabling the file garbage collector. If set to false, files mapped by + collectable mappers (such as the concurrent mapper) will not be deleted when their + swift variables go out of scope.""" +} + +mappingCheckerEnabled { + _type: "?Boolean" + _default: true + _doc: + """Controls the run-time duplicate mapping checker (which indetifies mapping + conflicts). When enabled, a record of all mapped data is kept, so this comes at the + expense of a slight memory leak. If set 'false', the mapping checker is disabled. + Enabled by default.""" +} + +hostName { + _type: "?String" + _default: null + _doc: + """Can be used to specify a publicly reacheable DNS name or IP address for this + machine which is generally used for Globus callbacks. Normally this should be + auto-detected, but if you do not have a public DNS name, you may want to set this.""" +} + +TCPPortRange { + _type: "?PortRange" + _default: null + _doc: + """A TCP port range can be specified to restrict the ports on which certain callback + services are started. This is likely needed if your submit host is behind a firewall, + in which case the firewall should be configured to allow incoming connections on + ports in this range.""" +} + +alwaysTransferWrapperLog { + _type: "?Boolean" + _default: false + _doc: + """Controls when wrapper logs are transfered back to the submit host. If set to + "false", Swift will only transfer a wrapper log for a given job when that job fails. + If set to "true", Swift will transfer wrapper logs whether a job fails or not.""" +} + +tracingEnabled { + _type: "?Boolean" + _default: false + _doc: + """Enables execution tracing. If set to "true", operations within Swift such as + iterations, invocations, assignments, and declarations, as well as data dependencies + will be logged. This comes at a cost in performance. It is therefore disabled by + default.""" +} + +wrapperStagingLocalServer { + _type: "?String" + _default: "file://" +} + +maxForeachThreads { + _type: "?StrictlyPositiveInt" + _default: 16384 + _doc_disabled: + """Limits the number of concurrent iterations that each foreach statement + can have at one time. This conserves memory for swift programs that + have large numbers of iterations (which would otherwise all be executed + in parallel).""" +} + +maxThreads { + _type: "?StrictlyPositiveInt" + _default: 1000000 +} \ No newline at end of file From hategan at ci.uchicago.edu Wed Jul 9 16:52:36 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Wed, 9 Jul 2014 16:52:36 -0500 (CDT) Subject: [Swift-commit] r7981 - in trunk: . lib Message-ID: <20140709215236.E2F3A9D66E@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-09 16:52:36 -0500 (Wed, 09 Jul 2014) New Revision: 7981 Added: trunk/lib/config-1.2.1.jar Modified: trunk/project.properties Log: added config jar Added: trunk/lib/config-1.2.1.jar =================================================================== (Binary files differ) Property changes on: trunk/lib/config-1.2.1.jar ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Modified: trunk/project.properties =================================================================== --- trunk/project.properties 2014-07-09 21:27:56 UTC (rev 7980) +++ trunk/project.properties 2014-07-09 21:52:36 UTC (rev 7981) @@ -8,4 +8,4 @@ # stable branch should be a change of version number) version = svn project = Swift workflow system -lib.deps = antlr*.*, jsr173_1.0_api.*, resolver.*, stringtemplate.*, vdldefinitions.*, xbean.*, xbean_xpath.*, jug*.*, jaxrpc*.*, jline*.jar, jfreechart*.jar, jcommon*.jar +lib.deps = antlr*.*, jsr173_1.0_api.*, resolver.*, stringtemplate.*, vdldefinitions.*, xbean.*, xbean_xpath.*, jug*.*, jaxrpc*.*, jline*.jar, jfreechart*.jar, jcommon*.jar, config*.jar From hategan at ci.uchicago.edu Thu Jul 10 19:45:04 2014 From: hategan at ci.uchicago.edu (hategan at ci.uchicago.edu) Date: Thu, 10 Jul 2014 19:45:04 -0500 (CDT) Subject: [Swift-commit] r7982 - trunk/docs/userguide Message-ID: <20140711004504.40B6F178884@svn.ci.uchicago.edu> Author: hategan Date: 2014-07-10 19:45:00 -0500 (Thu, 10 Jul 2014) New Revision: 7982 Added: trunk/docs/userguide/configuration.new trunk/docs/userguide/configuration.old Modified: trunk/docs/userguide/userguide.txt Log: added new configuration documentation Added: trunk/docs/userguide/configuration.new =================================================================== --- trunk/docs/userguide/configuration.new (rev 0) +++ trunk/docs/userguide/configuration.new 2014-07-11 00:45:00 UTC (rev 7982) @@ -0,0 +1,642 @@ +Configuration +------------- + +Swift is mainly configured using a configuration file, typically called *swift.conf*. +This file contains configuration properties and site descriptions. A simple +configuration file may look like this: + +----- +# include default Swift configuration file +include "${swift.home}/etc/swift.conf" + +site.mysite { + execution { + type: "coaster" + URL: "my.site.org" + jobManager: "ssh:local" + } + staging: "proxy" + + app.ALL {executable: "*"} +} + +# select sites to run on +sites: [mysite] + +# other settings +lazy.errors: false +----- + +Configuration Syntax +~~~~~~~~~~~~~~~~~~~~ + +The Swift configuration files are expressed in a modified version of JSON. The main +additions to JSON are: + +- Quotes around string values, in particular keys, are optional, unless the strings +contain special characters (single/double quotes, square and curly braces, white space, ++$+, +:+, +=+, +,+, +`+, +^+, +?+, +!+, + at +, +*+, +\+), or if they +represent other values: +true+, +false+, +null+, and numbers. +- +=+ and +:+ can be used interchangeably to separate keys from values +- +=+ (or +:+) is optional before an open bracket +- Commas are optional as separators if there is a new line +- +${...}+ expansion can be used to substitute environment variable values or Java + system properties. If the value of an environment variable is needed, it must be + prefixed with +env.+. For example +${env.PATH}+. Except for include directives, the + +${...}+ must not be inside double quotes for the substitution to work. The same + outcome can be achieved using implicit string concatenation: +"/home/"${env.USER}"/bin"+ + +Comments can be introduced by starting a line with a hash symbol (+#+) or using +a double slash (+//+): + +----- +# This is a comment +// This is also a comment + +keepSitesDir: true # This is a comment following a valid property +----- + +Include Directives +~~~~~~~~~~~~~~~~~~ + +Include directives can be used to include the contents of a Swift configuration file +from another Swift configuration file. This is done using the literal +include+ followed +by a quoted string containing the path to the target file. The path may contain +references to environment variables or system properties using the substitution +syntax explained above. For example: + +----- +# an absolute path name +include "/home/joedoe/swift-config/site1.conf" + +# include a file from the Swift distribution package +include "${swift.home}/etc/sites/beagle.conf" + +# include a file using an environment variable +include "${env.SWIFT_CONFIG_DIR}/glow.conf" +----- + +Configuration File Structure +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The contents of a Swift configuration file can be divided into a number of relevant +sections: + +- site declarations +- global application declarations +- Swift configuration properties + +Site Delcarations +^^^^^^^^^^^^^^^^^ + +Swift site declarations are specified using the +site.+ property, where text +inside angle brackets is to be interpreted as a generic label for user-specified +content, whereas content between square brackets is optional: + +----- +site. { + execution {...} + [staging: "swift" | "local" | "service-local" | "shared-fs" + | "proxy" | "wrapper"] + [filesystem {...}] + + [] + [] +} +----- + +A site name can be any string. If the string contains special characters, it must be +quoted: + +----- +site."My-$pecial-$ite" {...} +----- + +Execution Mechanism ++++++++++++++++++++ + +The +execution+ property tells Swift how applications should be executed on a site: + +----- + execution { + type: + [URL: ] + [jobManager: ] + + [] + } +----- + +The +type+ property is used to select one of the mechanisms for application execution +that is known by Swift. A comprehensinve list of execution mechanisms can be found +in <>. A summary is shown below: + +[[table-execution-mechanisms]] +.Swift Execution Mechanisms +[options="header",cols="3,2,2,2,4,10"] +|======================================================================================================================== +|Type |URL required|Uses jobManager|Default jobManager|Staging methods supported | +Description + +|+local+ | no | no | - | swift, local, wrapper | +Runs applications locally using a simple fork()-based mechanism + +|+coaster+ | yes | yes | none | swift, wrapper, service-local, shared-fs, proxy | +Submits applications through an automatically-deployed Swift Coasters service + +|+coaster-persistent+ | yes | yes | none | swift, wrapper, service-local, shared-fs, proxy | +Uses a manually deployed Swift Coasters service + +|+GRAM5+ | yes | yes | "fork" | swift, wrapper | +Uses the <> component of +the Globus Toolkit. + +|+GT2+ 4+| | +An alias for 'GRAM5' + +|+SSH+ | yes | no | - | swift, wrapper | +Runs applications using a Java implementation of the 'SSH' protocol + +|+SSH-CL+ | yes | no | - | swift, wrapper | +Like 'SSH' except it uses the command-line 'ssh' tool. + +|+PBS+ | no | no | - | swift, wrapper | +Submits applications to a PBS or Torque resource manager + +|+Condor+ | no | no | - | swift, wrapper | +Submits applications using Condor + +|+SGE+ | no | no | - | swift, wrapper | +Uses the Sun Grid Engine + +|+SLURM+ | no | no | - | swift, wrapper | +Uses the SLURM local scheduler + +|+LSF+ | no | no | - | swift, wrapper | +Submits applications to Platform's Load Sharing Facility + +|======================================================================================================================== + +The execution provider +options+ are options that specify finer details on how +on application should be executed. They depend on the chosen mechanism and are detailed in +<>. This is where Coasters options, such as +nodeGranularity+ or +softImage+, would +be specified. Example: + +----- +execution { + type: "coaster" + jobManager: "local:local" + options { + maxJobs: 1 + tasksPerNode: 2 + workerLoggingLevel: TRACE + } +} +----- + +A complete list of Swift Coasters options can be found in <> + +Staging ++++++++ + +The staging property instructs Swift how to handle application input and output files. +The 'swift' and 'wrapper' staging methods are supported universally, but the 'swift' method +requires the +filesystem+ property to be specified. If not specified, this option defaults to +'swift'. Support for the other choices is dependent on the execution mechanism. This is +detailed in the <> above. A +description of each staging method is provided in the table below: + +[[table-staging-methods]] +.Swift Staging Methods +[options="header",cols="3, 10"] +|============================================================================================= +| Staging Method | Description +| +swift+ | This method instructs Swift to use a filesystem provider to direct all + necessary staging operations from the Swift client-side to the cluster + head node. If this method is used, the +workDirectory+ must point to + a head node path that is on a shared file system accessible by the + compute nodes. +| +wrapper+ | File staging is done by the Swift application wrapper +| +local+ | Used by non-remote type execution mechanisms. It implements simple file + staging by copying files. +| +service-local+ | This method instructs the execution mechanism provider to stage input and + output files from the remote site where the execution service is located. + For example, if a Coaster Service is started on the login node of a + cluster, the Coaster Service will perform the staging from a file system + on the login node to the compute node and back. +| +shared-fs+ | This method is used by Coasters to implement a simple staging mechanism in + which files are accessed using a shared filesystem that is accessible by + compute nodes +| +proxy+ | This method is also used by Coasters to stage files from/to the Swift + client side to compute nodes by proxying through the Coaster Service. +|============================================================================================== + + +File System ++++++++++++ + +The file system properties are used with +staging: "swift"+ to tell Swift how to access remote +file systems. Valid types are described below: + +[[table-filesystem-providers]] +.Swift File System Providers +[options="header",cols="3, 3, 10"] +|========================================================================== +| Type | URL required | Description +| +local+ | no | Copies files locally on the Swift client side +| +GSIFTP+ | yes | Accesses a remote file system using GridFTP +| +GridFTP+ | yes | An alias for +GSIFTP+ +| +SSH+ | yes | Uses the SCP protocol +|========================================================================== + + + +Site Options +++++++++++++ + +Site options control various aspects of how Swift handles application execution on a site. +All options except +workDirectory+ are optional. They are listed in the following table: + + +[[table-site-options]] +.Site Options +[options="header",cols="1, 1, 1, 10"] +|================================= +| Option | Valid values | Default value | +Description +| +OS+ | many |"INTEL32::LINUX" | +Can be used to tell Swift the type of the operating system +running on the remote site. By default, Swift assumes a +UNIX/Linux type OS. There is some limited support for +running under Windows, in which case this property must be +set to one of +"INTEL32::WINDOWS"+ or +"INTEL64::WINDOWS"+ + +| +workDirectory+ | path | - | +Points to a directory in which Swift can maintain a set of +files relevant to the execution of an application on the +site. By default, applications will be executed on the +compute nodes in a sub-directory of +workDirectory+, which +implies that +workDirectory+ must be accessible from the +compute nodes. + +| +scratch+ | path | - | +If specified, it instructs swift to run applications in +a directory different than +workDirectory+. Contrary to the +requirement for +workDirectory+, +scratch+ can point to +a file system local to compute nodes. This option is useful +if applications do intensive I/O on temporary files created +in their work directory, or if they access their input/output +files in a non-linear fashion. + +| +keepSiteDir+ | +true, false+ | +false+ | +If set to +true+, site application directories (i.e. +workDirectory+) +will not be cleaned up when Swift completes a run. This +can be useful for debugging. + +| +statusMode+ | +"files", "provider"+ | +"files"+| +Controls whether application exit codes are handled by the +execution mechanism or passed back to Swift by the Swift +wrapper script through files. Traditionally, Globus GRAM +did not use to return application exit codes. This has +changed in Globus Toolkit 5.x. However, some local scheduler +execution mechanisms, such as 'PBS', are still unable to +return application exit codes. In such cases, it is necessary +to pass the application exit codes back to Swift in files. +This comes at a slight price in performance, since a file +needs to be created, written to, and transferred back to +Swift for each application invocation. It is however also +the default, since it works in all cases. + +| +maxParallelTasks+ | integer | 2 | +The maximum number of concurrent application invocations +allowed on this site. + +| +initialParallelTasks+ | integer | 2 | +The limit on the number of concurrent application invocations +on this site when a Swift run is started. As invocations +complete successfully, the number of concurrent invocations +on the site is increased up to +maxParallelTasks+. +|================================= + +Additional, less frequently used options, are as follows: + +[[table-site-options-obscure]] +.Obscure options that you are unlikely to need to worry about +[options="header",cols="1, 1, 1, 10"] +|================================= +| Option | Valid values | Default value | +Description + +| +wrapperParameterMode+ | +"args", "files"+ | +"args"+ | +If set to +"files"+, Swift will, as much as possible, pass +application arguments through files. The applications will +be invoked normally, with their arguments in the +**argv+ +parameter to the +main()+ function. This can be useful if the +execution mechanism has limitations on the size of command +line arguments that can be passed through. An example of +execution mechanism exhibiting this problem is Condor. + +| +wrapperInterpreter+ | path | +"/bin/bash"+ or +"cscript.exe"+ on Windows | +Points to the interpreter used to run the Swift application +invocation wrapper + +| +wrapperScript+ | string | +"_swiftwrap"+ or +"_swiftwrap.vbs"+ on Windows | +Points to the Swift application invocation wrapper. The file +must exist in the 'libexec' directory in the Swift distribution + +| +wrapperInterpreterOptions+ | list of strings | +[]+ on UNIX/Linux or +["//Nologo"]+ on Windows | +Command line options to be passed to the wrapper interpreter + +| +cleanupCommand+ | string | +"/bin/rm"+ or +"cmd.exe"+ on Windows | +A command to use for the cleaning of site directories (unless ++keepSiteDir+ is set to +true+) at the end of a run. + +| +cleanupCommandOptions+ | list of strings | +["-rf"]+ or +["/C", "del", "/Q"]+ on Windows | +Arguments to pass to the cleanup command when cleaning up site +work directories + +| +delayBase+ | number | 2.0 | +Swift keeps a quality indicator for each site it runs applications +on. This is a number that gets increased for every successful +application invocation, and decreased for every failure. It then +uses this number in deciding which sites to run applications on +(when multiple sites are defined). If this number becomes very +low (a sign of repeated failures on a site), Swift implements +an exponential back-off that prevents jobs from being sent to a +site that continously fails them. +delayBase+ is the base for +that exponential back-off: + +delay = delayBase ^ (-score * 100)+ + +| +maxSubmitRate+ | positive number| - | +Some combinations of site and execution mechanisms may become +error prone if jobs are submitted too fast. This option can +be used to limit the submission rate. If set to some number N, +Swift will submit applications at a rate of at most N +per second. + +|================================= + + +Application Declarations +++++++++++++++++++++++++ + +Applications can either be declared globally, outside of a site declaration, +or specific to a site, inside a site declaration: + +------ +app.(|ALL) { + # global application + ... +} + +site. { + app.(|ALL) { + # site application + ... + } +} +------ + +A special application name, +ALL+, can be used to declare options for all +applications. When Swift attempts to run an application named +X+, it will +first look at site application declarations for +app.X+. If not found, it will +check if a site application declaration exists for +app.ALL+. The search will +continue with the global +app.X+ and then the global +all.ALL+ until a match +is found. It is possible that a specific application will only be declared +on a sub-set of all the sites and not globally. Swift will then only select +a site where the application is declared and will not attempt to run the +application on other sites. + +An application declaration takes the following form: + +----- +app. { + executable: (|"*") + [jobQueue: ] + [jobProject: ] + [maxWallTime: