[Swift-commit] r7976 - in trunk/src/org: globus/swift/catalog/site globus/swift/catalog/types globus/swift/data griphyn/vdl/karajan griphyn/vdl/karajan/functions griphyn/vdl/karajan/lib griphyn/vdl/karajan/lib/replication griphyn/vdl/karajan/lib/swiftscript griphyn/vdl/mapping griphyn/vdl/mapping/file griphyn/vdl/mapping/nodes griphyn/vdl/util
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Wed Jul 9 14:38:50 CDT 2014
Author: hategan
Date: 2014-07-09 14:38:50 -0500 (Wed, 09 Jul 2014)
New Revision: 7976
Added:
trunk/src/org/griphyn/vdl/util/ConfigTree.java
trunk/src/org/griphyn/vdl/util/ConvertConfig.java
trunk/src/org/griphyn/vdl/util/SwiftConfig.java
trunk/src/org/griphyn/vdl/util/SwiftConfigException.java
trunk/src/org/griphyn/vdl/util/SwiftConfigInfo.java
trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java
Removed:
trunk/src/org/griphyn/vdl/util/TriStateBoolean.java
trunk/src/org/griphyn/vdl/util/VDL2Config.java
trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java
Modified:
trunk/src/org/globus/swift/catalog/site/Application.java
trunk/src/org/globus/swift/catalog/site/SwiftContact.java
trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java
trunk/src/org/globus/swift/catalog/types/Arch.java
trunk/src/org/globus/swift/catalog/types/Os.java
trunk/src/org/globus/swift/catalog/types/SysInfo.java
trunk/src/org/globus/swift/data/Director.java
trunk/src/org/griphyn/vdl/karajan/Loader.java
trunk/src/org/griphyn/vdl/karajan/SwiftRootScope.java
trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java
trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java
trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java
trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java
trunk/src/org/griphyn/vdl/karajan/lib/CacheFunction.java
trunk/src/org/griphyn/vdl/karajan/lib/Execute.java
trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java
trunk/src/org/griphyn/vdl/karajan/lib/New.java
trunk/src/org/griphyn/vdl/karajan/lib/Operators.java
trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java
trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java
trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java
trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java
trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java
trunk/src/org/griphyn/vdl/karajan/lib/Throttled.java
trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java
trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java
trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java
trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java
trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java
trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java
Log:
new config mechanism III
Modified: trunk/src/org/globus/swift/catalog/site/Application.java
===================================================================
--- trunk/src/org/globus/swift/catalog/site/Application.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/globus/swift/catalog/site/Application.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -42,7 +42,7 @@
env.put(name, value);
}
- public void addProperty(String name, String value) {
+ public void addProperty(String name, Object value) {
if (properties == null) {
properties = new HashMap<String, Object>();
}
Modified: trunk/src/org/globus/swift/catalog/site/SwiftContact.java
===================================================================
--- trunk/src/org/globus/swift/catalog/site/SwiftContact.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/globus/swift/catalog/site/SwiftContact.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -55,16 +55,31 @@
public void setSiteCatalog(SwiftContactSet siteCatalog) {
this.siteCatalog = siteCatalog;
}
+
+ /**
+ * Get an application with the specified tr. Only this site
+ * is searched. Returns <code>null</code> if not found.
+ */
+ public Application getApplication(String tr) {
+ if (apps == null) {
+ return null;
+ }
+ else {
+ return apps.get(tr);
+ }
+ }
+ /**
+ * Find apps by searching in the following order:
+ * <ol>
+ * <li>host:tr</li>
+ * <li>host:*</li>
+ * <li>pool:tr</li>
+ * <li>pool:*</li>
+ * </ol>
+ */
public Application findApplication(String tr) {
- /*
- * Find apps in the following order:
- * host:tr
- * host:*
- * pool:tr
- * pool:*
- */
-
+
Application app = null;
if (apps != null) {
app = apps.get(tr);
Modified: trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java
===================================================================
--- trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/globus/swift/catalog/site/SwiftContactSet.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -49,4 +49,11 @@
}
return app;
}
+
+ public Map<String, Application> getApplications() {
+ if (apps == null) {
+ apps = new HashMap<String, Application>();
+ }
+ return apps;
+ }
}
Modified: trunk/src/org/globus/swift/catalog/types/Arch.java
===================================================================
--- trunk/src/org/globus/swift/catalog/types/Arch.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/globus/swift/catalog/types/Arch.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -17,91 +17,7 @@
package org.globus.swift.catalog.types;
-/**
- * This is an enumerated data class for the different types of architecture.
- *
- * @author Gaurang Mehta gmehta at isi.edu
- * @version $Revision: 1.5 $
- */
+public enum Arch {
+ INTEL32, INTEL64, SPARCV7, SPARCV9, AMD64;
-import java.io.Serializable;
-import java.util.HashMap;
-
-public class Arch
- implements Serializable {
- private String _value_;
- private static HashMap<String, Arch> _table_ = new HashMap<String, Arch>(5);
-
- protected Arch(String value) {
- _value_ = value;
- _table_.put(_value_, this);
- }
-
- private static final String _INTEL32 = "INTEL32";
- private static final String _INTEL64 = "INTEL64";
- private static final String _SPARCV7 = "SPARCV7";
- private static final String _SPARCV9 = "SPARCV9";
- private static final String _AMD64 = "AMD64";
-
- public static final Arch INTEL32 = new Arch(_INTEL32);
- public static final Arch INTEL64 = new Arch(_INTEL64);
- public static final Arch SPARCV7 = new Arch(_SPARCV7);
- public static final Arch SPARCV9 = new Arch(_SPARCV9);
- public static final Arch AMD64 = new Arch(_AMD64);
-
- public static final String err = "Error: Illegal Architecture defined. Please specify one of the predefined types \n [INTEL32, INTEL64, AMD64, SPARCV7, SPARCV9]";
-
- /**
- * Returns the value of the architecture as string.
- * @return String
- */
- public String getValue() {
- return _value_;
- }
-
- /**
- * Creates a new Arch Object givan a arch string.
- * @param value String
- * @throws IllegalStateException Throws Exception if the architecure is not defined in this class.
- * @return Arch
- */
- public static Arch fromValue(String value) throws IllegalStateException {
- Arch m_enum = _table_.get(value.toUpperCase());
- if (m_enum == null) {
- throw new IllegalStateException(err);
- }
- return m_enum;
- }
-
- /**
- * Creates a new Arch object given a arch string.
- * @param value String
- * @throws IllegalStateException Throws Exception if the architecure is not defined in this class.
- * @return Arch
- */
- public static Arch fromString(String value) throws IllegalStateException {
- return fromValue(value);
- }
-
- /**
- * Compares if a given Arch object is equal to this.
- * @param obj Object
- * @return boolean
- */
- public boolean equals(Object obj) {
- return (obj == this);
- }
-
- public int hashCode() {
- return toString().hashCode();
- }
-
- /**
- * Returns the string value of the architecture.
- * @return String
- */
- public String toString() {
- return _value_;
- }
-
}
Modified: trunk/src/org/globus/swift/catalog/types/Os.java
===================================================================
--- trunk/src/org/globus/swift/catalog/types/Os.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/globus/swift/catalog/types/Os.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -17,89 +17,6 @@
package org.globus.swift.catalog.types;
-/**
- * This is an enumerated data class for the different types of operating systems.
- *
- * @author Gaurang Mehta gmehta at isi.edu
- * @version $Revision: 1.4 $
- */
-
-import java.io.Serializable;
-import java.util.HashMap;
-
-public class Os
- implements Serializable {
- private String _value_;
- private static HashMap<String, Os> _table_ = new HashMap<String, Os>(5);
-
- protected Os(String value) {
- _value_ = value;
- _table_.put(_value_, this);
- }
-
- private static final String _LINUX = "LINUX";
- private static final String _SUNOS = "SUNOS";
- private static final String _AIX = "AIX";
- private static final String _WINDOWS = "WINDOWS";
-
- public static final Os LINUX = new Os(_LINUX);
- public static final Os SUNOS = new Os(_SUNOS);
- public static final Os AIX = new Os(_AIX);
- public static final Os WINDOWS = new Os(_WINDOWS);
-
- public static final String err = "Error: Illegal Operating System defined. Please specify one of the predefined types \n [LINUX, SUNOS, AIX, WINDOWS]";
-
- /**
- * Returns the value of the operating system as string.
- * @return String
- */
- public String getValue() {
- return _value_;
- }
-
- /**
- * Creates a new Os object given an os string.
- * @param value String
- * @throws IllegalStateException Throws Exception if the operating system is not defined in this class.
- * @return Os
- */
- public static Os fromValue(String value) throws IllegalStateException {
- Os m_enum = _table_.get(value.toUpperCase());
- if (m_enum == null) {
- throw new IllegalStateException(err);
- }
- return m_enum;
- }
-
- /**
- * Creates a new Os object given an os string.
- * @param value String
- * @throws IllegalStateException Throws Exception if the operating system is not defined in this class.
- * @return Os
- */
- public static Os fromString(String value) throws IllegalStateException {
- return fromValue(value);
- }
-
- /**
- * Compares if a given Os object is equal to this.
- * @param obj Object
- * @return boolean
- */
- public boolean equals(Object obj) {
- return (obj == this);
- }
-
- public int hashCode() {
- return toString().hashCode();
- }
-
- /**
- * Returns the string value of the operating system.
- * @return String
- */
- public String toString() {
- return _value_;
- }
-
+public enum Os {
+ LINUX, SUNOS, AIX, WINDOWS;
}
Modified: trunk/src/org/globus/swift/catalog/types/SysInfo.java
===================================================================
--- trunk/src/org/globus/swift/catalog/types/SysInfo.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/globus/swift/catalog/types/SysInfo.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -77,8 +77,8 @@
* @param glibc String
*/
public SysInfo(String arch, String os, String osversion, String glibc) {
- this.arch = (arch == null) ? Arch.INTEL32 : Arch.fromString(arch);
- this.os = (os == null) ? Os.LINUX : Os.fromString(os);
+ this.arch = (arch == null) ? Arch.INTEL32 : Arch.valueOf(arch);
+ this.os = (os == null) ? Os.LINUX : Os.valueOf(os);
this.osversion = (osversion == null || osversion.equals("") ) ?
null:
osversion;
@@ -92,9 +92,9 @@
if (system != null) {
String s1[] = system.split("::", 2);
if (s1.length == 2) {
- arch = Arch.fromString(s1[0]);
+ arch = Arch.valueOf(s1[0]);
String s2[] = s1[1].split(":", 3);
- os = Os.fromString(s2[0]);
+ os = Os.valueOf(s2[0]);
for (int i = 1; i < s2.length; i++) {
if (i == 1) {
osversion = s2[i];
Modified: trunk/src/org/globus/swift/data/Director.java
===================================================================
--- trunk/src/org/globus/swift/data/Director.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/globus/swift/data/Director.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -33,10 +33,10 @@
import java.util.regex.Pattern;
import org.apache.log4j.Logger;
-import org.globus.swift.data.policy.Policy;
import org.globus.swift.data.policy.Broadcast;
+import org.globus.swift.data.policy.Policy;
import org.globus.swift.data.util.LineReader;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
/**
* Manages CDM policies for files based on pattern matching.
@@ -103,22 +103,20 @@
/**
Read in the user-supplied CDM policy file.
*/
- public static void load(File file) throws IOException {
+ public static void load(File file, SwiftConfig config) throws IOException {
logger.debug("CDM file: " + file);
Director.policyFile = file;
List<String> list = LineReader.read(file);
for (String s : list)
addLine(s);
- init();
+ init(config);
}
- static void init() throws IOException
- {
- VDL2Config config = VDL2Config.getConfig();
+ static void init(SwiftConfig config) throws IOException
+ {
+ broadcastMode = config.getStringProperty("cdm.broadcast.mode");
+ logfile = config.getStringProperty("logfile");
- broadcastMode = config.getProperty("cdm.broadcast.mode");
- logfile = config.getProperty("logfile");
-
if (broadcastMode.equals("file"))
broadcasted.put("LOCAL_FILE", new HashSet<String>());
@@ -313,7 +311,7 @@
logger.debug("Policy file does not exist: " +
args[1]);
}
- load(policyFile);
+ load(policyFile, SwiftConfig.getDefault());
Policy policy = lookup(name);
logger.debug(name + ": " + policy);
} catch (Exception e) {
Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -35,9 +35,11 @@
import java.security.SecureRandom;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Enumeration;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -67,9 +69,8 @@
import org.griphyn.vdl.toolkit.VDLt2VDLx.IncorrectInvocationException;
import org.griphyn.vdl.toolkit.VDLt2VDLx.ParsingException;
import org.griphyn.vdl.util.LazyFileAppender;
-import org.griphyn.vdl.util.VDL2Config;
-import org.griphyn.vdl.util.VDL2ConfigProperties;
-import org.griphyn.vdl.util.VDL2ConfigProperties.PropInfo;
+import org.griphyn.vdl.util.SwiftConfig;
+import org.griphyn.vdl.util.SwiftConfigSchema;
public class Loader extends org.globus.cog.karajan.Loader {
private static final Logger logger = Logger.getLogger(Loader.class);
@@ -78,18 +79,37 @@
public static final String ARG_VERSION = "version";
public static final String ARG_RESUME = "resume";
public static final String ARG_INSTANCE_CONFIG = "config";
+ public static final String ARG_SITELIST = "sitelist";
+ public static final String ARG_SITES_FILE = "sites.file";
+ public static final String ARG_TC_FILE = "tc.file";
public static final String ARG_DRYRUN = "dryrun";
public static final String ARG_VERBOSE = "verbose";
public static final String ARG_DEBUG = "debug";
public static final String ARG_LOGFILE = "logfile";
- public static final String ARG_CDMFILE = "cdm.file";
+ public static final String ARG_CDMFILE = "cdmfile";
public static final String ARG_RUNID = "runid";
public static final String ARG_UI = "ui";
public static final String ARG_RECOMPILE = "recompile";
- public static final String ARG_REDUCED_LOGGING = "reduced.logging";
- public static final String ARG_MINIMAL_LOGGING = "minimal.logging";
- public static final String ARG_PAUSE_ON_START = "pause.on.start";
+ public static final String ARG_REDUCED_LOGGING = "reducedLogging";
+ public static final String ARG_MINIMAL_LOGGING = "minimalLogging";
+ public static final String ARG_PAUSE_ON_START = "pauseOnStart";
public static final String ARG_EXECUTE = "e";
+
+ public static final List<String> CMD_LINE_OPTIONS;
+
+ static {
+ CMD_LINE_OPTIONS = new ArrayList<String>();
+ CMD_LINE_OPTIONS.add("hostName");
+ CMD_LINE_OPTIONS.add("TCPPortRange");
+ CMD_LINE_OPTIONS.add("lazyErrors");
+ CMD_LINE_OPTIONS.add("keepSiteDir");
+ CMD_LINE_OPTIONS.add("alwaysTransferWrapperLog");
+ CMD_LINE_OPTIONS.add("logProvenance");
+ CMD_LINE_OPTIONS.add("fileGCEnabled");
+ CMD_LINE_OPTIONS.add("mappingCheckerEnabled");
+ CMD_LINE_OPTIONS.add("tracingEnabled");
+ CMD_LINE_OPTIONS.add("maxForeachThreads");
+ }
public static String buildVersion;
@@ -120,18 +140,19 @@
source = null;
}
- VDL2Config config = loadConfig(ap);
- addCommandLineProperties(config, ap);
- config.validateProperties();
- setupLogging(ap, projectName, runID);
+ SwiftConfig config = loadConfig(ap, getCommandLineProperties(ap));
+ if (ap.isPresent(ARG_SITELIST)) {
+ printSiteList(config);
+ System.exit(0);
+ }
+ setupLogging(ap, config, projectName, runID);
logBasicInfo(argv, runID, config);
- debugSitesText(config);
- debugTCText(config);
+ debugConfigText(config);
- boolean provenanceEnabled = VDL2Config.getConfig().getProvenanceLog();
+ boolean provenanceEnabled = config.isProvenanceEnabled();
if (ap.isPresent(ARG_CDMFILE)) {
- loadCDM(ap);
+ loadCDM(ap, config);
}
WrapperNode tree = null;
@@ -235,7 +256,14 @@
System.exit(runerror ? 2 : 0);
}
- private static void logBasicInfo(String[] argv, String runID, VDL2Config conf) {
+ private static void printSiteList(SwiftConfig config) {
+ System.out.println("Available sites: ");
+ for (String name : config.getDefinedSiteNames()) {
+ System.out.println("\t" + name);
+ }
+ }
+
+ private static void logBasicInfo(String[] argv, String runID, SwiftConfig conf) {
String version = loadVersion();
System.out.println(version);
System.out.println("RunID: " + runID);
@@ -278,6 +306,18 @@
System.out.println(loadVersion());
System.exit(0);
}
+ if (ap.isPresent(ARG_SITES_FILE)) {
+ System.err.println("Swift does not use site files any more.\n" +
+ "Please use the swift-convert-config tool to update your " +
+ "sites file to a swift configuration file.");
+ System.exit(1);
+ }
+ if (ap.isPresent(ARG_TC_FILE)) {
+ System.err.println("Swift does not use TC files any more.\n" +
+ "Please use the swift-convert-config tool to update your " +
+ "sites.xml and tc.data to a Swift configuration file.");
+ System.exit(1);
+ }
if (!ap.hasValue(ArgumentParser.DEFAULT) && !ap.isPresent(ARG_EXECUTE)) {
System.out.println(loadVersion());
error("No Swift script specified");
@@ -328,13 +368,13 @@
System.err.print("For usage information: swift -help\n\n");
}
- static void loadCDM(ArgumentParser ap) {
+ static void loadCDM(ArgumentParser ap, SwiftConfig config) {
String cdmString = null;
try {
cdmString = ap.getStringValue(ARG_CDMFILE);
File cdmFile = new File(cdmString);
debugText("CDM FILE", cdmFile);
- Director.load(cdmFile);
+ Director.load(cdmFile, config);
}
catch (IOException e) {
logger.debug("Detailed exception:", e);
@@ -496,19 +536,10 @@
}
}
- static void debugSitesText(VDL2Config config) {
- String poolFile = config.getPoolFile();
- logger.info("SITES_FILE " + poolFile);
- debugText("SITES", new File(poolFile));
+ static void debugConfigText(SwiftConfig config) {
+ logger.info("SWIFT_CONF \n" + config);
}
- static void debugTCText(VDL2Config config) {
- String tcFile = config.getTCFile();
- logger.info("TC_FILE " + tcFile);
- debugText("TC", new File(tcFile));
- }
-
-
/**
* The build ID is a UID that gets generated with each build. It is
* used to decide whether an already compiled swift script can be
@@ -530,29 +561,30 @@
}
}
- private static VDL2Config loadConfig(ArgumentParser ap) throws IOException {
- VDL2Config conf;
+ private static SwiftConfig loadConfig(ArgumentParser ap, Map<String, Object> cmdLine) throws IOException {
+ SwiftConfig conf;
if (ap.hasValue(ARG_INSTANCE_CONFIG)) {
String configFile = ap.getStringValue(ARG_INSTANCE_CONFIG);
- conf = VDL2Config.getConfig(configFile);
+ conf = SwiftConfig.load(configFile, cmdLine);
+ SwiftConfig.setDefault(conf);
}
else {
- conf = (VDL2Config) VDL2Config.getConfig().clone();
+ conf = (SwiftConfig) SwiftConfig.load().clone();
}
return conf;
}
- private static void addCommandLineProperties(VDL2Config config,
- ArgumentParser ap) {
- config.setCurrentFile("<command line>");
- Map<String, PropInfo> desc = VDL2ConfigProperties.getPropertyDescriptions();
- for (Map.Entry<String, PropInfo> e : desc.entrySet()) {
+ private static Map<String, Object> getCommandLineProperties(ArgumentParser ap) {
+ Map<String, Object> cmdConf = new HashMap<String, Object>();
+ Map<String, SwiftConfigSchema.Info> desc = SwiftConfig.SCHEMA.getPropertyDescriptions();
+ for (Map.Entry<String, SwiftConfigSchema.Info> e : desc.entrySet()) {
String name = e.getKey();
if (ap.isPresent(name)) {
String value = ap.getStringValue(name);
- config.setProperty(name, value);
+ cmdConf.put(name, value);
}
}
+ return cmdConf;
}
private static ArgumentParser buildArgumentParser() {
@@ -574,6 +606,8 @@
ap.addFlag(ARG_DRYRUN,
"Runs the SwiftScript program without submitting any jobs (can be used to get a graph)");
+ ap.addHiddenFlag(ARG_SITES_FILE);
+ ap.addHiddenFlag(ARG_TC_FILE);
ap.addOption(ARG_RESUME, "Resumes the execution using a log file",
"file", ArgumentParser.OPTIONAL);
@@ -583,6 +617,7 @@
"If individual command line arguments are used for properties, they will override " +
"the contents of this file.", "file",
ArgumentParser.OPTIONAL);
+ ap.addFlag(ARG_SITELIST, "Prints a list of sites available in the swift configuration");
ap.addFlag(ARG_VERBOSE,
"Increases the level of output that Swift produces on the console to include more detail " +
"about the execution");
@@ -621,10 +656,10 @@
ap.addOption(ARG_EXECUTE, "Runs the swift script code contained in <string>",
"string", ArgumentParser.OPTIONAL);
- Map<String, PropInfo> desc = VDL2ConfigProperties.getPropertyDescriptions();
- for (Map.Entry<String, PropInfo> e : desc.entrySet()) {
- PropInfo pi = e.getValue();
- ap.addOption(e.getKey(), pi.desc, pi.validValues,
+ Map<String, SwiftConfigSchema.Info> desc = SwiftConfig.SCHEMA.getPropertyDescriptions();
+ for (Map.Entry<String, SwiftConfigSchema.Info> e : desc.entrySet()) {
+ SwiftConfigSchema.Info pi = e.getValue();
+ ap.addOption(e.getKey(), pi.doc, pi.type.toString(),
ArgumentParser.OPTIONAL);
}
return ap;
@@ -632,7 +667,7 @@
private static MonitorAppender ma;
- protected static void setupLogging(ArgumentParser ap, String projectName,
+ protected static String setupLogging(ArgumentParser ap, SwiftConfig config, String projectName,
String runID) throws IOException {
String logfile;
if (ap.isPresent(ARG_LOGFILE)) {
@@ -641,10 +676,9 @@
else {
logfile = projectName + "-" + runID + ".log";
}
-
- VDL2Config config = VDL2Config.getConfig();
- config.put("logfile", logfile);
+ config.setProperty("logfile", logfile);
+
File f = new File(logfile);
FileAppender fa = (FileAppender) getAppender(FileAppender.class);
@@ -698,6 +732,7 @@
Logger.getLogger(New.class).setLevel(Level.WARN);
Logger.getLogger("org.globus.cog.karajan.workflow.service").setLevel(Level.WARN);
}
+ return logfile;
}
Modified: trunk/src/org/griphyn/vdl/karajan/SwiftRootScope.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/SwiftRootScope.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/SwiftRootScope.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -17,14 +17,14 @@
import org.globus.cog.karajan.util.KarajanProperties;
import org.griphyn.vdl.karajan.lib.swiftscript.FnArg;
import org.griphyn.vdl.mapping.DuplicateMappingChecker;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class SwiftRootScope extends RootScope {
public SwiftRootScope(KarajanProperties props, String file, Context context) {
super(props, file, context);
context.setAttribute("SWIFT:DM_CHECKER", new DuplicateMappingChecker(
- (VDL2Config) context.getAttribute("SWIFT:CONFIG")));
+ (SwiftConfig) context.getAttribute("SWIFT:CONFIG")));
addVar("PATH_SEPARATOR", File.separator);
addVar("SWIFT:DRY_RUN", context.getAttribute("SWIFT:DRY_RUN"));
Modified: trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/VDSAdaptiveScheduler.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -52,6 +52,7 @@
import org.globus.cog.karajan.util.ContactSet;
import org.globus.cog.karajan.util.TypeUtil;
import org.globus.swift.catalog.site.SwiftContact;
+import org.griphyn.vdl.util.SwiftConfig;
public class VDSAdaptiveScheduler extends WeightedHostScoreScheduler implements CoasterResourceTracker {
@@ -78,7 +79,7 @@
serviceContactMapping = new HashMap<Service, BoundContact>();
}
- public static final String PROP_TC_FILE = "transformationCatalogFile";
+ public static final String PROP_CONFIG = "config";
public static final String PROP_CLUSTERING_ENABLED = "clusteringEnabled";
public static final String PROP_CLUSTERING_QUEUE_DELAY = "clusteringQueueDelay";
public static final String PROP_CLUSTERING_MIN_TIME = "clusteringMinTime";
@@ -88,15 +89,15 @@
public synchronized String[] getPropertyNames() {
if (propertyNames == null) {
propertyNames = AbstractScheduler.combineNames(super.getPropertyNames(),
- new String[] { PROP_TC_FILE });
+ new String[] { PROP_CONFIG });
}
return propertyNames;
}
public void setProperty(String name, Object value) {
- if (PROP_TC_FILE.equals(name)) {
+ if (PROP_CONFIG.equals(name)) {
this.setConstraintChecker(new SwiftSiteChecker());
- this.addTaskTransformer(new VDSTaskTransformer());
+ this.addTaskTransformer(new VDSTaskTransformer((SwiftConfig) value));
}
else if (PROP_CLUSTERING_QUEUE_DELAY.equals(name)) {
clusteringQueueDelay = TypeUtil.toInt(value);
Modified: trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/VDSTaskTransformer.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -20,7 +20,6 @@
*/
package org.griphyn.vdl.karajan;
-import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -34,15 +33,16 @@
import org.globus.cog.karajan.scheduler.TaskTransformer;
import org.globus.cog.karajan.util.BoundContact;
import org.globus.cog.karajan.util.Contact;
-import org.griphyn.vdl.util.VDL2Config;
+import org.globus.swift.catalog.site.SwiftContact;
+import org.griphyn.vdl.util.SwiftConfig;
public class VDSTaskTransformer implements TaskTransformer {
public static final Logger logger = Logger.getLogger(VDSTaskTransformer.class);
private TaskTransformer impl;
- public VDSTaskTransformer() {
- this.impl = new SwiftTransformer();
+ public VDSTaskTransformer(SwiftConfig config) {
+ this.impl = new SwiftTransformer(config);
}
public void transformTask(Task task, Contact[] contacts, Service[] services) {
@@ -50,8 +50,13 @@
}
public static abstract class AbstractTransformer implements TaskTransformer {
+ private SwiftConfig config;
- public void transformTask(Task task, Contact[] contacts, Service[] services) {
+ public AbstractTransformer(SwiftConfig config) {
+ this.config = config;
+ }
+
+ public void transformTask(Task task, Contact[] contacts, Service[] services) {
if (task.getType() == Task.JOB_SUBMISSION) {
applyJobWorkDirectory(task, contacts);
}
@@ -117,7 +122,7 @@
private void applyJobWorkDirectory(Task task, Contact[] contacts) {
JobSpecification spec = (JobSpecification) task.getSpecification();
String dir = spec.getDirectory();
- BoundContact bc = (BoundContact) contacts[0];
+ SwiftContact bc = (SwiftContact) contacts[0];
String workdir = (String) bc.getProperty("workdir");
if (workdir==null){
@@ -141,24 +146,24 @@
// undergo this substitution...
String executable = l.get(0);
- try {
- VDL2Config config = VDL2Config.getConfig();
+ String mode = (String) bc.getProperty(SwiftConfig.Key.WRAPPER_INVOCATION_MODE.propName);
+ if (mode == null) {
+ mode = config.getWrapperInvocationMode();
+ }
+ if (mode.equals("absolute")
+ && (executable.endsWith("shared/_swiftwrap")
+ || executable.endsWith("shared/_swiftseq"))) {
- if (config.getProperty("wrapper.invocation.mode", bc).equals("absolute")
- && (executable.endsWith("shared/_swiftwrap")
- || executable.endsWith("shared/_swiftseq"))) {
-
- String s = spec.getDirectory() + "/" + executable;
- l.set(0, s);
- }
- }
- catch(IOException ioe) {
- throw new RuntimeException("Could not determine wrapper invocation mode", ioe);
+ String s = spec.getDirectory() + "/" + executable;
+ l.set(0, s);
}
}
}
public static class SwiftTransformer extends AbstractTransformer {
+ public SwiftTransformer(SwiftConfig config) {
+ super(config);
+ }
}
}
Modified: trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -20,8 +20,6 @@
*/
package org.griphyn.vdl.karajan.functions;
-import java.io.IOException;
-
import k.rt.Context;
import k.rt.ExecutionException;
import k.rt.Stack;
@@ -39,7 +37,7 @@
import org.globus.cog.karajan.compiled.nodes.Node;
import org.globus.cog.karajan.parser.WrapperNode;
import org.globus.cog.karajan.util.BoundContact;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class ConfigProperty extends InternalFunction {
private ArgRef<String> name;
@@ -47,7 +45,7 @@
private ArgRef<BoundContact> host;
private ChannelRef<Object> cr_vargs;
- private VDL2Config instanceConfig;
+ private SwiftConfig instanceConfig;
private VarRef<Context> context;
@@ -68,9 +66,12 @@
context = scope.getVarRef("#context");
Context ctx = context.getValue();
Var.Channel r = scope.parent.lookupChannel("...");
+ if (this.name.isStatic()) {
+ checkProperty(this.name.getValue());
+ }
if (this.name.isStatic() && this.instance.isStatic() && this.host.isStatic()
&& ctx != null && this.host.getValue() == null) {
- String value = getProperty(this.name.getValue(), this.instance.getValue(), getInstanceConfig(ctx));
+ Object value = getProperty(this.name.getValue(), this.instance.getValue(), getInstanceConfig(ctx));
if (r.append(value)) {
return null;
}
@@ -108,44 +109,45 @@
cr_vargs.append(stack, getProperty(name, instance, getInstanceConfig(stack)));
}
- private synchronized VDL2Config getInstanceConfig(Stack stack) {
+ private synchronized SwiftConfig getInstanceConfig(Stack stack) {
if (instanceConfig == null) {
Context ctx = this.context.getValue(stack);
- instanceConfig = (VDL2Config) ctx.getAttribute("SWIFT:CONFIG");
+ instanceConfig = (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG");
}
return instanceConfig;
}
- private synchronized VDL2Config getInstanceConfig(Context ctx) {
- return (VDL2Config) ctx.getAttribute("SWIFT:CONFIG");
+ private synchronized SwiftConfig getInstanceConfig(Context ctx) {
+ return (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG");
}
- public static String getProperty(String name, VDL2Config instanceConfig) {
+ public static Object getProperty(String name, SwiftConfig instanceConfig) {
return getProperty(name, true, instanceConfig);
}
- public static String getProperty(String name, boolean instance, VDL2Config instanceConfig) {
- try {
- VDL2Config conf;
- String prop;
- if (!instance) {
- conf = VDL2Config.getConfig();
- prop = conf.getProperty(name);
- }
- else {
- conf = instanceConfig;
- prop = conf.getProperty(name);
- }
- if (prop == null) {
- throw new ExecutionException("Swift config property \"" + name + "\" not found in "
- + conf);
- }
- else {
- return prop;
- }
+ public static Object getProperty(String name, boolean instance, SwiftConfig instanceConfig) {
+ SwiftConfig conf;
+ Object prop;
+ if (!instance) {
+ conf = SwiftConfig.getDefault();
+ prop = conf.getProperty(name);
}
- catch (IOException e) {
- throw new ExecutionException("Failed to load Swift configuration", e);
+ else {
+ conf = instanceConfig;
+ prop = conf.getProperty(name);
}
+ if (prop == null) {
+ throw new ExecutionException("Swift config property \"" + name + "\" not found in "
+ + conf.getFileName());
+ }
+ else {
+ return prop;
+ }
}
+
+ private void checkProperty(String name) throws CompilationException {
+ if (!SwiftConfig.SCHEMA.propertyExists(name)) {
+ throw new CompilationException(this, "Unknown configuration property: " + name);
+ }
+ }
}
Modified: trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/functions/ProcessBulkErrors.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -76,7 +76,10 @@
VDL2ErrorTranslator translator = VDL2ErrorTranslator.getDefault();
Map<String, Integer> count = new HashMap<String, Integer>();
- for (ExecutionException ex : l) {
+ for (ExecutionException ex : l) {
+ if (ex == null) {
+ continue;
+ }
if (ex.getCause() instanceof ConcurrentModificationException) {
ex.printStackTrace();
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/CacheFunction.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/CacheFunction.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/CacheFunction.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -26,14 +26,12 @@
import org.globus.cog.karajan.analyzer.CompilationException;
import org.globus.cog.karajan.analyzer.Scope;
import org.globus.cog.karajan.analyzer.VarRef;
-import org.globus.cog.karajan.compiled.nodes.Node;
import org.globus.cog.karajan.compiled.nodes.InternalFunction;
+import org.globus.cog.karajan.compiled.nodes.Node;
import org.globus.cog.karajan.parser.WrapperNode;
-import org.griphyn.vdl.karajan.functions.ConfigProperty;
import org.griphyn.vdl.karajan.lib.cache.VDLFileCache;
import org.griphyn.vdl.karajan.lib.cache.VDLFileCacheFactory;
-import org.griphyn.vdl.util.VDL2Config;
-import org.griphyn.vdl.util.VDL2ConfigProperties;
+import org.griphyn.vdl.util.SwiftConfig;
public abstract class CacheFunction extends InternalFunction {
public static final String CACHE_FILES_TO_REMOVE = "cacheFilesToRemove";
@@ -62,8 +60,8 @@
synchronized(ctx) {
VDLFileCache cache = (VDLFileCache) ctx.getAttribute("SWIFT:FILE_CACHE");
if (cache == null) {
- cache = VDLFileCacheFactory.newInstance(ConfigProperty.getProperty(
- VDL2ConfigProperties.CACHING_ALGORITHM, (VDL2Config) ctx.getAttribute("SWIFT:CONFIG")));
+ SwiftConfig conf = (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG");
+ cache = VDLFileCacheFactory.newInstance(conf.getCachingAlgorithm());
ctx.setAttribute("SWIFT:FILE_CACHE", cache);
}
return cache;
Modified: trunk/src/org/griphyn/vdl/karajan/lib/Execute.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Execute.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -52,7 +52,7 @@
import org.griphyn.vdl.karajan.lib.RuntimeStats.ProgressState;
import org.griphyn.vdl.karajan.lib.replication.CanceledReplicaException;
import org.griphyn.vdl.karajan.lib.replication.ReplicationManager;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class Execute extends GridExec {
public static final Logger logger = Logger.getLogger(Execute.class);
@@ -66,6 +66,7 @@
private VarRef<Context> context;
private boolean replicationEnabled;
+ private SwiftConfig config;
@Override
protected Signature getSignature() {
@@ -103,8 +104,8 @@
protected void addLocals(Scope scope) {
super.addLocals(scope);
context = scope.getVarRef("#context");
- VDL2Config config = (VDL2Config) context.getValue().getAttribute("SWIFT:CONFIG");
- replicationEnabled = "true".equals(config.getProperty("replication.enabled"));
+ config = (SwiftConfig) context.getValue().getAttribute("SWIFT:CONFIG");
+ replicationEnabled = config.isReplicationEnabled();
}
@Override
@@ -232,7 +233,7 @@
synchronized (ctx) {
ReplicationManager rm = (ReplicationManager) ctx.getAttribute("#replicationManager");
if (rm == null) {
- rm = new ReplicationManager(getScheduler(stack));
+ rm = new ReplicationManager(getScheduler(stack), config);
ctx.setAttribute("#replicationManager", rm);
}
return rm;
Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/GetURLPrefix.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -27,8 +27,7 @@
import org.globus.cog.karajan.analyzer.Scope;
import org.globus.cog.karajan.analyzer.VarRef;
import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction;
-import org.griphyn.vdl.karajan.functions.ConfigProperty;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class GetURLPrefix extends AbstractSingleValuedFunction {
private VarRef<Context> context;
@@ -49,8 +48,8 @@
@Override
public Object function(Stack stack) {
Context ctx = this.context.getValue(stack);
- String localServerBase = ConfigProperty.getProperty("wrapper.staging.local.server",
- (VDL2Config) ctx.getAttribute("SWIFT:CONFIG"));
+ SwiftConfig config = (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG");
+ String localServerBase = config.getWrapperStagingLocalServer();
String cwd = this.cwd.getValue(stack);
if (cwd.endsWith("/.")) {
Modified: trunk/src/org/griphyn/vdl/karajan/lib/New.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/New.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/New.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -59,9 +59,7 @@
public static final Logger logger = Logger.getLogger(New.class);
private static final Mapper NULL_MAPPER = new NullMapper();
-
- private static DuplicateMappingChecker staticDMC = new DuplicateMappingChecker(null);
-
+
private ArgRef<Field> field;
private ArgRef<GenericMappingParamSet> mapping;
private ArgRef<Object> value;
Modified: trunk/src/org/griphyn/vdl/karajan/lib/Operators.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Operators.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Operators.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -28,22 +28,11 @@
import org.griphyn.vdl.type.Field;
import org.griphyn.vdl.type.Type;
import org.griphyn.vdl.type.Types;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class Operators {
- public static final boolean PROVENANCE_ENABLED;
-
- static {
- boolean v;
- try {
- v = VDL2Config.getConfig().getProvenanceLog();
- }
- catch (Exception e) {
- v = false;
- }
- PROVENANCE_ENABLED = v;
- }
+ public static final boolean PROVENANCE_ENABLED = SwiftConfig.getDefault().isProvenanceEnabled();
public static final Logger provenanceLogger = Logger.getLogger("org.globus.swift.provenance.operators");
Modified: trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Parameterlog.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -20,6 +20,7 @@
*/
package org.griphyn.vdl.karajan.lib;
+import k.rt.Context;
import k.rt.Stack;
import k.thr.LWThread;
@@ -29,11 +30,10 @@
import org.globus.cog.karajan.analyzer.Scope;
import org.globus.cog.karajan.analyzer.Signature;
import org.globus.cog.karajan.analyzer.VarRef;
+import org.globus.cog.karajan.compiled.nodes.InternalFunction;
import org.globus.cog.karajan.compiled.nodes.Node;
-import org.globus.cog.karajan.compiled.nodes.InternalFunction;
import org.globus.cog.karajan.parser.WrapperNode;
-import org.griphyn.vdl.karajan.functions.ConfigProperty;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class Parameterlog extends InternalFunction {
public static final Logger logger = Logger.getLogger(Parameterlog.class);
@@ -48,12 +48,12 @@
}
private Boolean enabled;
- private VarRef<VDL2Config> config;
+ private VarRef<Context> context;
@Override
protected Node compileBody(WrapperNode w, Scope argScope, Scope scope)
throws CompilationException {
- config = scope.getVarRef("SWIFT_CONFIG");
+ context = scope.getVarRef("#context");
return super.compileBody(w, argScope, scope);
}
@@ -65,7 +65,9 @@
boolean run;
synchronized(this) {
if (enabled == null) {
- enabled = "true".equals(ConfigProperty.getProperty("provenance.log", true, config.getValue(stack)));
+ Context ctx = this.context.getValue(stack);
+ SwiftConfig config = (SwiftConfig) ctx.getAttribute("SWIFT:CONFIG");
+ enabled = config.isProvenanceEnabled();
}
run = enabled;
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/RuntimeStats.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -17,7 +17,6 @@
package org.griphyn.vdl.karajan.lib;
-import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.HashMap;
@@ -27,7 +26,6 @@
import java.util.Set;
import k.rt.Context;
-import k.rt.ExecutionException;
import k.rt.Stack;
import k.thr.LWThread;
@@ -42,26 +40,15 @@
import org.globus.cog.karajan.compiled.nodes.InternalFunction;
import org.globus.cog.karajan.compiled.nodes.Node;
import org.globus.cog.karajan.parser.WrapperNode;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
/** this is an icky class that does too much with globals, but is for
proof of concept. */
public class RuntimeStats {
- public static final boolean TICKER_DISABLED;
+ public static final boolean TICKER_DISABLED = !SwiftConfig.getDefault().isTickerEnabled();
- static{
- boolean disabled;
- try{
- disabled = "true".equalsIgnoreCase(VDL2Config.getConfig().getProperty("ticker.disable"));
- }
- catch (Exception e) {
- disabled = false;
- }
- TICKER_DISABLED = disabled;
- }
-
public static final String TICKER = "SWIFT_TICKER";
//formatter for timestamp against std.err lines
@@ -105,13 +92,7 @@
t.start();
context.getValue(thr.getStack()).setAttribute(TICKER, t);
// Allow user to reformat output date
- String format;
- try {
- format = VDL2Config.getDefaultConfig().getTickerDateFormat();
- }
- catch (IOException e) {
- throw new ExecutionException(this, e);
- }
+ String format = SwiftConfig.getDefault().getTickerDateFormat();
if (format != null && format.length() > 0) {
formatter = new SimpleDateFormat(format);
}
@@ -253,17 +234,11 @@
public ProgressTicker() {
super("Progress ticker");
states = new HashSet<ProgressState>();
- try {
- if ("true".equalsIgnoreCase(VDL2Config.getConfig().getProperty("ticker.disable"))) {
- logger.info("Ticker disabled in configuration file");
- disabled = true;
- }
- tickerPrefix =
- VDL2Config.getConfig().getTickerPrefix();
+ if (!SwiftConfig.getDefault().isTickerEnabled()) {
+ logger.info("Ticker disabled in configuration file");
+ disabled = true;
}
- catch (IOException e) {
- logger.debug("Could not read swift properties", e);
- }
+ tickerPrefix =SwiftConfig.getDefault().getTickerPrefix();
start = System.currentTimeMillis();
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/SiteCatalog.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -9,412 +9,24 @@
*/
package org.griphyn.vdl.karajan.lib;
-import java.util.ArrayList;
-import java.util.List;
-
-import k.rt.ExecutionException;
import k.rt.Stack;
-import org.globus.cog.abstraction.impl.common.AbstractionFactory;
-import org.globus.cog.abstraction.impl.common.ProviderMethodException;
-import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl;
-import org.globus.cog.abstraction.impl.common.task.InvalidProviderException;
-import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl;
-import org.globus.cog.abstraction.impl.common.task.ServiceImpl;
-import org.globus.cog.abstraction.interfaces.ExecutionService;
-import org.globus.cog.abstraction.interfaces.Service;
-import org.globus.cog.abstraction.interfaces.ServiceContact;
import org.globus.cog.karajan.analyzer.ArgRef;
import org.globus.cog.karajan.analyzer.Param;
import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction;
-import org.globus.swift.catalog.site.Application;
-import org.globus.swift.catalog.site.SiteCatalogParser;
-import org.globus.swift.catalog.site.SwiftContact;
-import org.globus.swift.catalog.site.SwiftContactSet;
-import org.w3c.dom.Document;
-import org.w3c.dom.NamedNodeMap;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
+import org.griphyn.vdl.util.SwiftConfig;
public class SiteCatalog extends AbstractSingleValuedFunction {
- private ArgRef<String> fileName;
+ private ArgRef<SwiftConfig> config;
@Override
protected Param[] getParams() {
- return params("fileName");
+ return params("config");
}
@Override
public Object function(Stack stack) {
- String fn = fileName.getValue(stack);
- SiteCatalogParser p = new SiteCatalogParser(fn);
- try {
- Document doc = p.parse();
- return buildResources(doc);
- }
- catch (Exception e) {
- throw new ExecutionException(this, "Failed to parse site catalog", e);
- }
+ SwiftConfig config = this.config.getValue(stack);
+ return config.getSites();
}
-
- private static class KVPair {
- public final String key;
- public final String value;
-
- public KVPair(String key, String value) {
- this.key = key;
- this.value = value;
- }
- }
-
- private Object buildResources(Document doc) {
- Node root = getRoot(doc);
-
- if (root.getLocalName().equals("config")) {
- throw new IllegalArgumentException("Old sites file format. Please upgrade your sites file to the new format.");
- }
- else if (root.getLocalName().equals("sites")) {
- return parse(root);
- }
- else {
- throw new IllegalArgumentException("Illegal sites file root node: " + root.getLocalName());
- }
- }
-
- private Object parse(Node config) {
- SwiftContactSet cs = new SwiftContactSet();
- NodeList pools = config.getChildNodes();
- for (int i = 0; i < pools.getLength(); i++) {
- Node n = pools.item(i);
- if (n.getNodeType() == Node.ELEMENT_NODE) {
- String ctype = n.getNodeName();
- if (ctype.equals("site")) {
- try {
- SwiftContact bc = pool(n);
- if (bc != null) {
- cs.addContact(bc);
- }
- }
- catch (Exception e) {
- throw new ExecutionException(this, "Invalid site entry '" + poolName(n) + "': ", e);
- }
- }
- else if (ctype.equals("apps")) {
- SwiftContact dummy = new SwiftContact();
- apps(dummy, n);
- for (Application app : dummy.getApplications()) {
- cs.addApplication(app);
- }
- }
- else {
- throw new IllegalArgumentException("Invalid node: " + ctype);
- }
- }
- }
- return cs;
- }
-
- private String poolName(Node site) {
- if (site.getLocalName().equals("site")) {
- return attr(site, "name");
- }
- else {
- throw new IllegalArgumentException("Invalid node: " + site.getLocalName());
- }
- }
-
- private Node getRoot(Document doc) {
- NodeList l = doc.getChildNodes();
- for (int i = 0; i < l.getLength(); i++) {
- if (l.item(i).getNodeType() == Node.ELEMENT_NODE) {
- return l.item(i);
- }
- }
- throw new IllegalArgumentException("Missing root element");
- }
-
- private SwiftContact pool(Node n) throws InvalidProviderException, ProviderMethodException {
- if (n.getNodeType() != Node.ELEMENT_NODE) {
- return null;
- }
- String name = poolName(n);
- SwiftContact bc = new SwiftContact(name);
-
- String sysinfo = attr(n, "sysinfo", null);
- if (sysinfo != null) {
- bc.setProperty("sysinfo", sysinfo);
- }
-
- NodeList cs = n.getChildNodes();
-
- for (int i = 0; i < cs.getLength(); i++) {
- Node c = cs.item(i);
- if (c.getNodeType() != Node.ELEMENT_NODE) {
- continue;
- }
- String ctype = c.getNodeName();
-
- if (ctype.equals("execution")) {
- bc.addService(execution(c));
- }
- else if (ctype.equals("filesystem")) {
- bc.addService(filesystem(c));
- }
- else if (ctype.equals("workdirectory")) {
- bc.setProperty("workdir", text(c));
- }
- else if (ctype.equals("scratch")) {
- bc.setProperty("scratch", text(c));
- }
- else if (ctype.equals("property")) {
- bc.setProperty(attr(c, "name"), text(c));
- }
- else if (ctype.equals("apps")) {
- apps(bc, c);
- }
- else {
- throw new IllegalArgumentException("Unknown node type: " + ctype);
- }
- }
- return bc;
- }
-
- private void apps(SwiftContact bc, Node n) {
- NodeList cs = n.getChildNodes();
-
- List<KVPair> envs = new ArrayList<KVPair>();
- List<KVPair> props = new ArrayList<KVPair>();
- for (int i = 0; i < cs.getLength(); i++) {
- Node c = cs.item(i);
- if (c.getNodeType() != Node.ELEMENT_NODE) {
- continue;
- }
- String ctype = c.getNodeName();
-
- if (ctype.equals("app")) {
- bc.addApplication(application(c));
- }
- else if (ctype.equals("env")) {
- envs.add(env(c));
- }
- else if (ctype.equals("property")) {
- props.add(this.property(c));
- }
- else {
- throw new IllegalArgumentException("Unknown node type: " + ctype);
- }
- }
-
- mergeEnvsToApps(bc, envs);
- mergePropsToApps(bc, props);
- }
-
- private void mergeEnvsToApps(SwiftContact bc, List<KVPair> envs) {
- for (Application app : bc.getApplications()) {
- for (KVPair kvp : envs) {
- if (!app.getEnv().containsKey(kvp.key)) {
- // only merge if app does not override
- app.setEnv(kvp.key, kvp.value);
- }
- }
- }
- }
-
- private void mergePropsToApps(SwiftContact bc, List<KVPair> props) {
- for (Application app : bc.getApplications()) {
- for (KVPair kvp : props) {
- if (!app.getProperties().containsKey(kvp.key)) {
- app.addProperty(kvp.key, kvp.value);
- }
- }
- }
- }
-
- private Application application(Node n) {
- Application app = new Application();
- app.setName(attr(n, "name"));
- app.setExecutable(attr(n, "executable"));
-
- NodeList cs = n.getChildNodes();
-
- for (int i = 0; i < cs.getLength(); i++) {
- Node c = cs.item(i);
- if (c.getNodeType() != Node.ELEMENT_NODE) {
- continue;
- }
- String ctype = c.getNodeName();
-
- if (ctype.equals("env")) {
- KVPair env = env(c);
- app.setEnv(env.key, env.value);
- }
- else if (ctype.equals("property")) {
- KVPair prop = property(c);
- app.addProperty(prop.key, prop.value);
- }
- else {
- throw new IllegalArgumentException("Unknown node type: " + ctype);
- }
- }
-
- return app;
- }
-
- private Service execution(Node n) throws InvalidProviderException, ProviderMethodException {
- String provider = attr(n, "provider");
- String url = attr(n, "url", null);
- String jobManager = attr(n, "jobManager", null);
- if (jobManager == null) {
- jobManager = attr(n, "jobmanager", null);
- }
-
- ExecutionService s = new ExecutionServiceImpl();
- s.setProvider(provider);
- ServiceContact contact = null;
- if (url != null) {
- contact = new ServiceContactImpl(url);
- s.setServiceContact(contact);
- s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact));
- }
-
- if (jobManager != null) {
- s.setJobManager(jobManager);
- }
-
- properties(s, n);
-
- return s;
- }
-
- private void properties(Service s, Node n) {
- NodeList cs = n.getChildNodes();
-
- for (int i = 0; i < cs.getLength(); i++) {
- Node c = cs.item(i);
- if (c.getNodeType() != Node.ELEMENT_NODE) {
- continue;
- }
- String ctype = c.getNodeName();
-
- if (ctype.equals("property")) {
- property(s, c);
- }
- else {
- throw new IllegalArgumentException("Unknown node type: " + ctype);
- }
- }
-
- }
-
- private void property(Service s, Node c) {
- s.setAttribute(attr(c, "name"), text(c));
- }
-
- private KVPair property(Node c) {
- return new KVPair(attr(c, "name"), text(c));
- }
-
- private Service filesystem(Node n) throws InvalidProviderException, ProviderMethodException {
- String provider = attr(n, "provider");
- String url = attr(n, "url", null);
-
- Service s = new ServiceImpl();
- s.setType(Service.FILE_OPERATION);
- s.setProvider(provider);
-
- ServiceContact contact = null;
- if (url != null) {
- contact = new ServiceContactImpl(url);
- s.setServiceContact(contact);
- s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact));
- }
-
- properties(s, n);
-
- return s;
- }
-
- private KVPair env(Node n) {
- String key = attr(n, "name");
- String value = text(n);
-
- return new KVPair(key, value);
- }
-
- private String text(Node n) {
- if (n.getFirstChild() != null) {
- return n.getFirstChild().getNodeValue();
- }
- else {
- return null;
- }
- }
-
- private String attr(Node n, String name) {
- NamedNodeMap attrs = n.getAttributes();
- if (attrs != null) {
- Node attr = attrs.getNamedItem(name);
- if (attr == null) {
- throw new IllegalArgumentException("Missing " + name);
- }
- else {
- return expandProps(attr.getNodeValue());
- }
- }
- else {
- throw new IllegalArgumentException("Missing " + name);
- }
- }
-
- private String attr(Node n, String name, String defVal) {
- NamedNodeMap attrs = n.getAttributes();
- if (attrs != null) {
- Node attr = attrs.getNamedItem(name);
- if (attr == null) {
- return defVal;
- }
- else {
- return expandProps(attr.getNodeValue());
- }
- }
- else {
- return defVal;
- }
- }
-
- private String expandProps(String v) {
- if (v == null) {
- return null;
- }
- StringBuilder sb = new StringBuilder();
- int li = -1;
- for (int i = 0; i < v.length(); i++) {
- char c = v.charAt(i);
- switch (c) {
- case '{':
- if (li != -1) {
- li = -1;
- sb.append('{');
- }
- else {
- li = i;
- }
- break;
- case '}':
- if (li != -1) {
- sb.append(System.getProperty(v.substring(li + 1, i)));
- li = -1;
- }
- else {
- sb.append(c);
- }
- break;
- default:
- if (li == -1) {
- sb.append(c);
- }
- }
- }
- return sb.toString();
- }
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/SiteProperty.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -54,6 +54,8 @@
return getSingle(bc, name.getValue(stack), _default.getValue(stack));
}
+ public static final Os DEFAULT_OS = Os.LINUX;
+
public static final String SWIFT_WRAPPER_INTERPRETER = "wrapperInterpreter";
public static final String SWIFT_WRAPPER_INTERPRETER_OPTIONS = "wrapperInterpreterOptions";
public static final String SWIFT_WRAPPER_SCRIPT = "wrapperScript";
@@ -141,9 +143,9 @@
}
private Os getOS(SwiftContact bc) {
- Object o = bc.getProperty("sysinfo");
+ Object o = bc.getProperty("OS");
if (o == null) {
- return Os.LINUX;
+ return DEFAULT_OS;
}
else {
return SysInfo.fromString(o.toString()).getOs();
Modified: trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/SwiftFunction.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -17,7 +17,6 @@
package org.griphyn.vdl.karajan.lib;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -56,25 +55,13 @@
import org.griphyn.vdl.type.Field;
import org.griphyn.vdl.type.Type;
import org.griphyn.vdl.type.Types;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public abstract class SwiftFunction extends AbstractFunction {
public static final Logger logger = Logger.getLogger(SwiftFunction.class);
- public static final boolean PROVENANCE_ENABLED;
-
- static {
- boolean v;
- try {
- v = VDL2Config.getConfig().getProvenanceLog();
- }
- catch (IOException e) {
- v = false;
- }
- PROVENANCE_ENABLED = v;
- }
+ public static final boolean PROVENANCE_ENABLED = SwiftConfig.getDefault().isProvenanceEnabled();
-
private VarRef<Context> context;
@Override
Modified: trunk/src/org/griphyn/vdl/karajan/lib/Throttled.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Throttled.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Throttled.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -20,7 +20,6 @@
*/
package org.griphyn.vdl.karajan.lib;
-import java.io.IOException;
import java.util.LinkedList;
import k.rt.ConditionalYield;
@@ -29,8 +28,7 @@
import k.thr.Yield;
import org.globus.cog.karajan.compiled.nodes.Sequential;
-import org.globus.cog.karajan.util.TypeUtil;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class Throttled extends Sequential {
public static final int DEFAULT_MAX_THREADS = 1000000;
@@ -39,13 +37,7 @@
private int maxThreadCount, current;
public Throttled() {
- try {
- maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig()
- .getProperty("max.threads", String.valueOf(DEFAULT_MAX_THREADS)));
- }
- catch (IOException e) {
- maxThreadCount = DEFAULT_MAX_THREADS;
- }
+ maxThreadCount = (Integer) SwiftConfig.getDefault().getProperty("maxThreads", DEFAULT_MAX_THREADS);
current = 0;
waiting = new LinkedList<FutureObject>();
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/ThrottledParallelFor.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -17,7 +17,6 @@
package org.griphyn.vdl.karajan.lib;
-import java.io.IOException;
import java.util.Iterator;
import java.util.List;
@@ -41,8 +40,7 @@
import org.globus.cog.karajan.compiled.nodes.Node;
import org.globus.cog.karajan.compiled.nodes.UParallelFor;
import org.globus.cog.karajan.parser.WrapperNode;
-import org.globus.cog.karajan.util.TypeUtil;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class ThrottledParallelFor extends UParallelFor {
public static final Logger logger = Logger
@@ -229,13 +227,7 @@
private int getMaxThreads() {
if (maxThreadCount < 0) {
- try {
- maxThreadCount = TypeUtil.toInt(VDL2Config.getConfig()
- .getProperty("foreach.max.threads", String.valueOf(DEFAULT_MAX_THREADS)));
- }
- catch (IOException e) {
- maxThreadCount = DEFAULT_MAX_THREADS;
- }
+ maxThreadCount = SwiftConfig.getDefault().getForeachMaxThreads();
}
return maxThreadCount;
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/Tracer.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -9,7 +9,6 @@
*/
package org.griphyn.vdl.karajan.lib;
-import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -25,7 +24,7 @@
import org.griphyn.vdl.mapping.RootHandle;
import org.griphyn.vdl.mapping.nodes.AbstractDataNode;
import org.griphyn.vdl.type.Types;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class Tracer {
public static final Logger logger = Logger.getLogger("TRACE");
@@ -34,12 +33,7 @@
private ThreadLocal<String> thread = new ThreadLocal<String>();
static {
- try {
- globalTracingEnabled = VDL2Config.getConfig().isTracingEnabled();
- }
- catch (IOException e) {
- globalTracingEnabled = false;
- }
+ globalTracingEnabled = SwiftConfig.getDefault().isTracingEnabled();
NAME_MAPPINGS = new HashMap<String, String>();
NAME_MAPPINGS.put("assignment", "ASSIGN");
NAME_MAPPINGS.put("iterate", "ITERATE");
Modified: trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/replication/ReplicationManager.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -29,7 +29,7 @@
import org.globus.cog.abstraction.interfaces.JobSpecification;
import org.globus.cog.abstraction.interfaces.Task;
import org.globus.cog.karajan.scheduler.Scheduler;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class ReplicationManager {
public static final Logger logger = Logger
@@ -52,19 +52,15 @@
private ReplicationGroups replicationGroups;
private Scheduler scheduler;
- public ReplicationManager(Scheduler scheduler) {
+ public ReplicationManager(Scheduler scheduler, SwiftConfig config) {
this.replicationGroups = new ReplicationGroups(scheduler);
this.scheduler = scheduler;
queued = new HashMap<Task, Date>();
running = new HashMap<Task, Date>();
try {
- minQueueTime = Integer.parseInt(VDL2Config.getConfig().getProperty(
- "replication.min.queue.time"));
- enabled = Boolean.valueOf(
- VDL2Config.getConfig().getProperty("replication.enabled"))
- .booleanValue();
- limit = Integer.parseInt(VDL2Config.getConfig().getProperty(
- "replication.limit"));
+ minQueueTime = config.getReplicationMinQueueTime();
+ enabled = config.isReplicationEnabled();
+ limit = config.getReplicationLimit();
}
catch (Exception e) {
logger.warn(
Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -18,7 +18,6 @@
package org.griphyn.vdl.karajan.lib.swiftscript;
import java.io.BufferedReader;
-import java.io.IOException;
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.Map;
@@ -45,24 +44,13 @@
import org.griphyn.vdl.mapping.nodes.NodeFactory;
import org.griphyn.vdl.type.Field;
import org.griphyn.vdl.type.Types;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
public class Misc {
private static final Logger logger = Logger.getLogger(Misc.class);
- public static final boolean PROVENANCE_ENABLED;
-
- static {
- boolean v;
- try {
- v = VDL2Config.getConfig().getProvenanceLog();
- }
- catch (IOException e) {
- v = false;
- }
- PROVENANCE_ENABLED = v;
- }
+ public static final boolean PROVENANCE_ENABLED = SwiftConfig.getDefault().isProvenanceEnabled();
private static final Logger traceLogger = Logger.getLogger("org.globus.swift.trace");
Modified: trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/mapping/DuplicateMappingChecker.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -16,8 +16,7 @@
import org.apache.log4j.Logger;
import org.griphyn.vdl.mapping.nodes.AbstractDataNode;
-import org.griphyn.vdl.util.VDL2Config;
-import org.griphyn.vdl.util.VDL2ConfigProperties;
+import org.griphyn.vdl.util.SwiftConfig;
public class DuplicateMappingChecker {
public static final Logger logger = Logger.getLogger(DuplicateMappingChecker.class);
@@ -26,13 +25,8 @@
private final Map<PhysicalFormat, Entry> map;
- public DuplicateMappingChecker(VDL2Config conf) {
- if (conf == null) {
- enabled = true;
- }
- else {
- enabled = !"off".equals(conf.getProperty(VDL2ConfigProperties.DM_CHECKER));
- }
+ public DuplicateMappingChecker(SwiftConfig conf) {
+ enabled = conf.isMappingCheckerEnabled();
map = new HashMap<PhysicalFormat, Entry>();
}
Modified: trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/mapping/file/FileGarbageCollector.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -17,7 +17,6 @@
package org.griphyn.vdl.mapping.file;
-import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -27,8 +26,7 @@
import org.apache.log4j.Logger;
import org.griphyn.vdl.mapping.PhysicalFormat;
-import org.griphyn.vdl.util.VDL2Config;
-import org.griphyn.vdl.util.VDL2ConfigProperties;
+import org.griphyn.vdl.util.SwiftConfig;
public class FileGarbageCollector implements Runnable {
public static final Logger logger = Logger.getLogger(FileGarbageCollector.class);
@@ -52,13 +50,7 @@
queue = new LinkedList<PhysicalFormat>();
usageCount = new HashMap<PhysicalFormat, Integer>();
persistent = new HashSet<PhysicalFormat>();
- try {
- enabled = !"false".equals(VDL2Config.getConfig().getProperty(VDL2ConfigProperties.FILE_GC_ENABLED));
- }
- catch (IOException e) {
- //enabled by default
- enabled = true;
- }
+ enabled = SwiftConfig.getDefault().isFileGCEnabled();
if (enabled) {
thread = new Thread(this, "File Garbage Collector");
thread.setDaemon(true);
Modified: trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/mapping/nodes/AbstractDataNode.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -20,7 +20,6 @@
*/
package org.griphyn.vdl.mapping.nodes;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -44,7 +43,7 @@
import org.griphyn.vdl.mapping.PhysicalFormat;
import org.griphyn.vdl.type.Field;
import org.griphyn.vdl.type.Type;
-import org.griphyn.vdl.util.VDL2Config;
+import org.griphyn.vdl.util.SwiftConfig;
@@ -79,11 +78,7 @@
public static boolean provenance = false;
static {
- try {
- provenance = VDL2Config.getConfig().getProvenanceLog();
- }
- catch (IOException e) {
- }
+ provenance = SwiftConfig.getDefault().isProvenanceEnabled();
}
protected Field field;
Modified: trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -10,25 +10,46 @@
package org.griphyn.vdl.util;
import java.io.File;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import java.util.SortedSet;
import java.util.TreeSet;
-public abstract class ConfigPropertyType {
- public static final ConfigPropertyType BOOLEAN = choices("true", "false");
- public static final ConfigPropertyType ONOFF = choices("on", "off");
- public static final ConfigPropertyType STRING = new CPTString();
- public static final ConfigPropertyType INT = new Int();
- public static final ConfigPropertyType FLOAT = new CPTFloat();
- public static final ConfigPropertyType FILE = new CPTFile();
+import org.globus.cog.abstraction.impl.common.execution.WallTime;
+import org.globus.swift.catalog.types.Arch;
+import org.globus.swift.catalog.types.Os;
+
+public abstract class ConfigPropertyType<T> {
+ public static final ConfigPropertyType<Boolean> BOOLEAN = new CPTBoolean();
+ public static final ConfigPropertyType<String> STRING = new CPTString();
+ public static final ConfigPropertyType<Integer> INT = new Int();
+ public static final ConfigPropertyType<Object> THROTTLE = new Throttle();
+ public static final ConfigPropertyType<String> PORT_RANGE = new PortRange();
+ public static final ConfigPropertyType<Integer> STRICTLY_POSITIVE_INT = new SPInt();
+ public static final ConfigPropertyType<Integer> POSITIVE_INT = new PInt();
+ public static final ConfigPropertyType<Double> POSITIVE_FLOAT = new PFloat();
+ public static final ConfigPropertyType<Double> FLOAT = new CPTFloat();
+ public static final ConfigPropertyType<String> FILE = new CPTFile();
+ public static final ConfigPropertyType<String> TIME = new CPTTime();
+ public static final ConfigPropertyType<Object> OBJECT = new CPTObject();
+ public static final ConfigPropertyType<Object> STRING_LIST = new StringList();
+ public static final ConfigPropertyType<String> OS = new CPTOS();
- public static ConfigPropertyType choices(String... values) {
+ public static ConfigPropertyType<String> choices(String... values) {
return new Choices(values);
}
- public abstract void checkValue(String propName, String value, String source);
+ @SuppressWarnings("unchecked")
+ public Object check(String propName, Object value, String source) {
+ return checkValue(propName, (T) value, source);
+ }
+ public abstract Object checkValue(String propName, T value, String source);
+
+ public abstract ConfigPropertyType<?> getBaseType();
+
private static String pp(Collection<String> c) {
StringBuilder sb = new StringBuilder();
Iterator<String> i = c.iterator();
@@ -43,8 +64,8 @@
return sb.toString();
}
- private static class Choices extends ConfigPropertyType {
- private SortedSet<String> choices;
+ public static class Choices extends ConfigPropertyType<String> {
+ protected SortedSet<String> choices;
public Choices(String... values) {
choices = new TreeSet<String>();
@@ -54,55 +75,343 @@
}
@Override
- public void checkValue(String propName, String value, String source) {
+ public Object checkValue(String propName, String value, String source) {
if (!choices.contains(value)) {
throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
propName + "'. Valid values are: " + pp(choices));
}
+ return value;
}
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return STRING;
+ }
+
+ @Override
+ public String toString() {
+ return "one of " + choices;
+ }
}
- private static class CPTString extends ConfigPropertyType {
+ private static class CPTOS extends Choices {
+ public CPTOS() {
+ super();
+ int ix = 0;
+ for (Arch a : Arch.values()) {
+ for (Os o : Os.values()) {
+ choices.add(a + "::" + o);
+ }
+ }
+ }
+ }
+
+ private static class CPTString extends ConfigPropertyType<String> {
@Override
- public void checkValue(String propName, String value, String source) {
+ public Object checkValue(String propName, String value, String source) {
// all values accepted
+ return value;
}
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return STRING;
+ }
+
+ @Override
+ public String toString() {
+ return "string";
+ }
}
- private static class Int extends ConfigPropertyType {
+ private static class Int extends ConfigPropertyType<Integer> {
@Override
- public void checkValue(String propName, String value, String source) {
- try {
- Integer.parseInt(value);
+ public Object checkValue(String propName, Integer value, String source) {
+ return value;
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return INT;
+ }
+
+ @Override
+ public String toString() {
+ return "integer";
+ }
+ }
+
+ private static class CPTBoolean extends ConfigPropertyType<Boolean> {
+ @Override
+ public Object checkValue(String propName, Boolean value, String source) {
+ return value;
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return BOOLEAN;
+ }
+
+ @Override
+ public String toString() {
+ return "boolean";
+ }
+ }
+
+ private static class SPInt extends ConfigPropertyType<Integer> {
+ @Override
+ public Object checkValue(String propName, Integer value, String source) {
+ if (value <= 0) {
+ throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
+ propName + "'. Must be a " + toString());
}
- catch (NumberFormatException e) {
+ return value;
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return INT;
+ }
+
+ @Override
+ public String toString() {
+ return "strictly positive integer";
+ }
+ }
+
+ private static class PInt extends ConfigPropertyType<Integer> {
+ @Override
+ public Object checkValue(String propName, Integer value, String source) {
+ if (value < 0) {
throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
- propName + "'. Must be an integer");
+ propName + "'. Must be a " + toString());
}
+ return value;
}
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return INT;
+ }
+
+ @Override
+ public String toString() {
+ return "positive integer";
+ }
}
+
- private static class CPTFloat extends ConfigPropertyType {
+ private static class Throttle extends ConfigPropertyType<Object> {
@Override
- public void checkValue(String propName, String value, String source) {
- try {
- Double.parseDouble(value);
+ public Object checkValue(String propName, Object value, String source) {
+ if ("off".equals(value)) {
+ return Integer.MAX_VALUE;
}
- catch (NumberFormatException e) {
+ else if (value instanceof Integer) {
+ Integer i = (Integer) value;
+ if (i > 0) {
+ return i;
+ }
+ }
+ throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
+ propName + "'. Must be an " + toString());
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return INT;
+ }
+
+ @Override
+ public String toString() {
+ return "integer greater than zero or \"off\"";
+ }
+ }
+
+ private static class PFloat extends ConfigPropertyType<Double> {
+ @Override
+ public Object checkValue(String propName, Double value, String source) {
+ if (value < 0) {
throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
- propName + "'. Must be a floating point number.");
+ propName + "'. Must be a " + toString());
}
+ return value;
}
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return FLOAT;
+ }
+
+ @Override
+ public String toString() {
+ return "positive number";
+ }
}
- private static class CPTFile extends ConfigPropertyType {
+ private static class CPTFloat extends ConfigPropertyType<Double> {
@Override
- public void checkValue(String propName, String value, String source) {
+ public Object checkValue(String propName, Double value, String source) {
+ return value;
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return FLOAT;
+ }
+ }
+
+ public static class Interval extends ConfigPropertyType<Double> {
+ private double l, h;
+
+ public Interval(double l, double h) {
+ this.l = l;
+ this.h = h;
+ }
+
+ @Override
+ public Object checkValue(String propName, Double value, String source) {
+ if (value < l || value > h) {
+ throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
+ propName + "'. Must be a " + toString());
+ }
+ return value;
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return FLOAT;
+ }
+
+ @Override
+ public String toString() {
+ return "floating point number in the interval [" + l + ", " + h + "]";
+ }
+ }
+
+ private static class CPTTime extends ConfigPropertyType<String> {
+ @Override
+ public Object checkValue(String propName, String value, String source) {
+ try {
+ WallTime.timeToSeconds(value);
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(source + ":\n\tInvalid time value '" + value + "' for property '" +
+ propName + "'. Mist be a " + toString());
+ }
+ return value;
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return STRING;
+ }
+
+ @Override
+ public String toString() {
+ return "string in one of the formats MM, HH:MM, or HH:MM:SS";
+ }
+ }
+
+ private static class PortRange extends ConfigPropertyType<String> {
+ @Override
+ public Object checkValue(String propName, String value, String source) {
+ String[] els = value.split(",\\s*");
+ if (els.length == 2) {
+ try {
+ Integer.parseInt(els[0]);
+ Integer.parseInt(els[1]);
+ return value;
+ }
+ catch (NumberFormatException e) {
+ }
+ }
+ throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
+ propName + "'. Must be a " + toString());
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return STRING;
+ }
+
+ @Override
+ public String toString() {
+ return "port range in the format 'port1, port2'";
+ }
+ }
+
+ private static class CPTFile extends ConfigPropertyType<String> {
+ @Override
+ public Object checkValue(String propName, String value, String source) {
File f = new File(value);
if (!f.exists()) {
throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
propName + "'. File does not exist.");
}
+ return value;
}
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return STRING;
+ }
+
+ @Override
+ public String toString() {
+ return "file path";
+ }
}
+
+ private static class CPTObject extends ConfigPropertyType<Object> {
+ @Override
+ public Object checkValue(String propName, Object value, String source) {
+ return value;
+ }
+
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return OBJECT;
+ }
+
+ @Override
+ public String toString() {
+ return "object";
+ }
+ }
+
+ private static class StringList extends ConfigPropertyType<Object> {
+ @Override
+ public Object checkValue(String propName, Object value, String source) {
+ if (value instanceof List) {
+ List<?> l = (List<?>) value;
+ boolean allStrings = true;
+ for (Object o : l) {
+ if (!(o instanceof String)) {
+ allStrings = false;
+ }
+ }
+ if (allStrings) {
+ return value;
+ }
+ }
+ else if (value instanceof String) {
+ // also allow comma separated strings in a string
+ return Arrays.asList(((String) value).split(",\\s*"));
+ }
+ throw new IllegalArgumentException(source + ":\n\tInvalid value '" + value + "' for property '" +
+ propName + "'. Must be a " + toString());
+ }
+
+ @Override
+ public ConfigPropertyType<?> getBaseType() {
+ return OBJECT;
+ }
+
+ @Override
+ public String toString() {
+ return "list of strings";
+ }
+ }
}
Added: trunk/src/org/griphyn/vdl/util/ConfigTree.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/ConfigTree.java (rev 0)
+++ trunk/src/org/griphyn/vdl/util/ConfigTree.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -0,0 +1,319 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 6, 2014
+ */
+package org.griphyn.vdl.util;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+public class ConfigTree<T> {
+
+ public static class Node<T> {
+ private Map<String, Node<T>> nodes;
+ private T value;
+
+ protected void checkEmpty(String k) {
+ if (k.isEmpty()) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ protected String first(String k) {
+ int ix = k.indexOf('.');
+ if (ix == -1) {
+ return k;
+ }
+ else {
+ return k.substring(0, ix);
+ }
+ }
+
+ protected String rest(String k) {
+ int ix = k.indexOf('.');
+ if (ix == -1) {
+ return "";
+ }
+ else {
+ return k.substring(ix + 1);
+ }
+ }
+
+ public Object get() {
+ return value;
+ }
+
+ public T get(String k) {
+ return get(k, null);
+ }
+
+ public T get(String k, String wildcard) {
+ if (k.isEmpty()) {
+ return value;
+ }
+ if (nodes == null || nodes.isEmpty()) {
+ throw new NoSuchElementException();
+ }
+ Node<T> t = nodes.get(first(k));
+ if (t == null && wildcard != null) {
+ t = nodes.get(wildcard);
+ }
+ if (t == null) {
+ throw new NoSuchElementException();
+ }
+ return t.get(rest(k), wildcard);
+ }
+
+ public boolean hasKey(String k) {
+ if (k.isEmpty()) {
+ return value != null;
+ }
+ if (nodes == null || nodes.isEmpty()) {
+ return false;
+ }
+ Node<T> t = nodes.get(first(k));
+ if (t == null) {
+ return false;
+ }
+ return t.hasKey(rest(k));
+ }
+
+ public T put(String k, T v) {
+ if (k.isEmpty()) {
+ return set(v);
+ }
+ else {
+ if (nodes == null) {
+ nodes = new HashMap<String, Node<T>>();
+ }
+ String first = first(k);
+ if (k == first) {
+ return getOrCreateTree(k).set(v);
+ }
+ else {
+ return getOrCreateTree(first).put(rest(k), v);
+ }
+ }
+ }
+
+ private Node<T> getOrCreateTree(String k) {
+ Node<T> t = nodes.get(k);
+ if (t == null) {
+ t = new Node<T>();
+ nodes.put(k, t);
+ return t;
+ }
+ else {
+ return t;
+ }
+ }
+
+ public void getLeafPaths(List<String> l, String partial) {
+ if (value != null) {
+ l.add(partial);
+ }
+ if (nodes != null) {
+ for (Map.Entry<String, Node<T>> e : nodes.entrySet()) {
+ if (partial == null) {
+ e.getValue().getLeafPaths(l, e.getKey());
+ }
+ else {
+ e.getValue().getLeafPaths(l, partial + "." + e.getKey());
+ }
+ }
+ }
+ }
+
+ public void expandWildcards(List<String> l, String k, String wildcard, String partial) {
+ if (value != null) {
+ if (partial == null) {
+ l.add(k);
+ }
+ else {
+ l.add(partial + "." + k);
+ }
+ }
+ if (nodes == null || nodes.isEmpty()) {
+ if (rest(k).isEmpty()) {
+ if (partial == null) {
+ l.add(k);
+ }
+ else {
+ l.add(partial + "." + k);
+ }
+ }
+ return;
+ }
+ String mk = first(k);
+ if (mk.equals(wildcard)) {
+ for (Map.Entry<String, Node<T>> e : nodes.entrySet()) {
+ Node<T> n = e.getValue();
+ String rest = rest(k);
+ String p;
+ if (partial == null) {
+ p = e.getKey();
+ }
+ else {
+ p = partial + "." + e.getKey();
+ }
+ n.expandWildcards(l, rest, wildcard, p);
+ }
+ }
+ else {
+ Node<T> t = nodes.get(mk);
+ if (t == null) {
+ if (first(rest(k)).equals(wildcard)) {
+ // x.* is allowed to not be there
+ return;
+ }
+ if (partial == null) {
+ l.add(k);
+ }
+ else {
+ l.add(partial + "." + k);
+ }
+ return;
+ }
+ String rest = rest(k);
+ String p;
+ if (partial == null) {
+ p = mk;
+ }
+ else {
+ p = partial + "." + mk;
+ }
+ t.expandWildcards(l, rest, wildcard, p);
+ }
+ }
+
+ public T set(T v) {
+ T old = value;
+ value = v;
+ return old;
+ }
+
+ public Set<Map.Entry<String, Node<T>>> entrySet() {
+ if (nodes == null) {
+ Map<String, Node<T>> empty = Collections.emptyMap();
+ return empty.entrySet();
+ }
+ else {
+ return nodes.entrySet();
+ }
+ }
+
+ private void toString(StringBuilder sb, int level, String k) {
+ for (int i = 0; i < level; i++) {
+ sb.append('\t');
+ }
+ if (nodes == null || nodes.isEmpty()) {
+ if (value != null) {
+ sb.append(k);
+ sb.append(": ");
+ if (value instanceof String) {
+ sb.append('\"');
+ sb.append(value);
+ sb.append('\"');
+ }
+ else {
+ sb.append(value);
+ }
+ sb.append('\n');
+ }
+ }
+ else if (nodes.size() == 1) {
+ String key = nodes.keySet().iterator().next();
+ if (k == null) {
+ nodes.values().iterator().next().toString(sb, 0, key);
+ }
+ else {
+ nodes.values().iterator().next().toString(sb, 0, k + "." + key);
+ }
+ }
+ else {
+ if (k != null) {
+ sb.append(k);
+ sb.append(' ');
+ }
+ sb.append("{\n");
+ for (Map.Entry<String, Node<T>> e : nodes.entrySet()) {
+ e.getValue().toString(sb, level + 1, e.getKey());
+ }
+ for (int i = 0; i < level; i++) {
+ sb.append('\t');
+ }
+ sb.append("}\n");
+ }
+ }
+ }
+
+ private Node<T> root;
+
+ public ConfigTree() {
+ root = new Node<T>();
+ }
+
+ public T get(String k) {
+ return get(k, null);
+ }
+
+ public T put(String k, T v) {
+ return root.put(k, v);
+ }
+
+ public T get(String k, String wildcard) {
+ try {
+ return root.get(k, wildcard);
+ }
+ catch (IllegalArgumentException e) {
+ throw new NoSuchElementException("Not a leaf: " + k);
+ }
+ catch (NoSuchElementException e) {
+ return null;
+ }
+ }
+
+ public List<String> getLeafPaths() {
+ List<String> l = new ArrayList<String>();
+ root.getLeafPaths(l, null);
+ return l;
+ }
+
+ /**
+ * Find all paths matching the given path. Wildcards are expanded based
+ * on what's in the tree, but the full paths do not need to exist in the tree.
+ *
+ * So if a.1.b.2 and a.1.b.3 were in the tree, a.*.b.*.c would generate
+ * a.1.b.2.c and a.1.b.3.c
+ *
+ */
+ public List<String> expandWildcards(String key, String wildcard) {
+ List<String> l = new ArrayList<String>();
+ root.expandWildcards(l, key, wildcard, null);
+ return l;
+ }
+
+ public boolean hasKey(String k) {
+ return root.hasKey(k);
+ }
+
+ public Set<Map.Entry<String, Node<T>>> entrySet() {
+ return root.entrySet();
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ root.toString(sb, 0, null);
+ return sb.toString();
+ }
+}
Added: trunk/src/org/griphyn/vdl/util/ConvertConfig.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/ConvertConfig.java (rev 0)
+++ trunk/src/org/griphyn/vdl/util/ConvertConfig.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -0,0 +1,772 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jan 7, 2013
+ */
+package org.griphyn.vdl.util;
+
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.globus.cog.abstraction.impl.common.AbstractionFactory;
+import org.globus.cog.abstraction.impl.common.ProviderMethodException;
+import org.globus.cog.abstraction.impl.common.execution.WallTime;
+import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl;
+import org.globus.cog.abstraction.impl.common.task.InvalidProviderException;
+import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl;
+import org.globus.cog.abstraction.impl.common.task.ServiceImpl;
+import org.globus.cog.abstraction.interfaces.ExecutionService;
+import org.globus.cog.abstraction.interfaces.Service;
+import org.globus.cog.abstraction.interfaces.ServiceContact;
+import org.globus.cog.karajan.scheduler.WeightedHost;
+import org.globus.cog.karajan.util.BoundContact;
+import org.globus.cog.karajan.util.ContactSet;
+import org.globus.cog.karajan.util.TypeUtil;
+import org.globus.cog.util.ArgumentParser;
+import org.globus.cog.util.ArgumentParserException;
+import org.globus.swift.catalog.TCEntry;
+import org.globus.swift.catalog.TransformationCatalog;
+import org.globus.swift.catalog.site.SiteCatalogParser;
+import org.globus.swift.catalog.transformation.File;
+import org.globus.swift.catalog.util.Profile;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+public class ConvertConfig {
+
+ private enum Version {
+ V1, V2;
+ }
+
+ private ContactSet parseSites(String fileName) {
+ SiteCatalogParser p = new SiteCatalogParser(fileName);
+ try {
+ Document doc = p.parse();
+ return buildResources(doc);
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Failed to parse site catalog", e);
+ }
+ }
+
+ private TransformationCatalog loadTC(String fileName) {
+ return File.getNonSingletonInstance(fileName);
+ }
+
+ public void run(String sitesFile, String tcFile, String conf, PrintStream ps) {
+ if (sitesFile == null && tcFile == null && conf == null) {
+ throw new IllegalArgumentException("You must specify at " +
+ "least one file type to convert");
+ }
+ ContactSet sites = null;
+ if (sitesFile != null) {
+ sites = parseSites(sitesFile);
+ }
+ TransformationCatalog tc = null;
+ if (tcFile != null) {
+ tc = loadTC(tcFile);
+ }
+
+ generateConfig(sites, tc, ps);
+
+ if (conf != null) {
+ Properties props = new Properties();
+ try {
+ props.load(new FileReader(conf));
+ generateConfig(props, ps);
+ }
+ catch (IOException e) {
+ throw new RuntimeException("Could not load config file", e);
+ }
+ }
+ }
+
+ private void generateConfig(Properties props, PrintStream ps) {
+ Set<String> processed = new HashSet<String>();
+ for (String name : propOrder) {
+ if (props.containsKey(name)) {
+ prop(name, props.get(name), ps);
+ processed.add(name);
+ }
+ }
+ for (Map.Entry<Object, Object> e : props.entrySet()) {
+ String name = (String) e.getKey();
+ if (!processed.contains(name)) {
+ prop(name, props.get(name), ps);
+ }
+ }
+ }
+
+ private void prop(String name, Object value, PrintStream ps) {
+ if (removedAttrs.contains(name)) {
+ ps.println("# option removed: " + name);
+ return;
+ }
+ if (name.equals("use.provider.staging")) {
+ if (TypeUtil.toBoolean(value)) {
+ ps.println("staging: \"provider\"");
+ }
+ return;
+ }
+ if (name.equals("use.wrapper.staging")) {
+ if (TypeUtil.toBoolean(value)) {
+ ps.println("staging: \"wrapper\"");
+ }
+ return;
+ }
+ Attr a = props.get(name);
+ if (a == null) {
+ throw new RuntimeException("Unknown configuration property '" + name + "'");
+ }
+ ps.print(a.name + ": ");
+ printValue(a.type, value, ps);
+ }
+
+ private void generateConfig(ContactSet sites, TransformationCatalog tc, PrintStream ps) {
+ ps.println("include \"${swift.home}/etc/swift.conf\"");
+ ps.println();
+
+ if (sites != null) {
+ for (BoundContact bc : sites.getContacts()) {
+ ps.println("site." + bc.getName() + " {");
+ boolean hasFileService = false;
+ boolean hasCoasterService = false;
+ for (Map.Entry<BoundContact.TypeProviderPair, Service> e : bc.getServices().entrySet()) {
+ String provider = e.getKey().provider;
+ if (provider == null) {
+ continue;
+ }
+ if (e.getKey().type == Service.EXECUTION) {
+ ExecutionService es = (ExecutionService) e.getValue();
+ ps.println("\texecution {");
+ ps.println("\t\ttype: \"" + provider + "\"");
+ if (es.getServiceContact() != null) {
+ ps.println("\t\tURL: \"" + es.getServiceContact().getContact() + "\"");
+ }
+ if (es.getJobManager() != null) {
+ ps.println("\t\tjobManager: \"" + es.getJobManager() + "\"");
+ }
+ if (provider.equals("coaster")) {
+ hasCoasterService = true;
+ generateCoasterServiceAttributes(bc, ps);
+ }
+ ps.println("\t}");
+ if (!provider.equals("coaster")) {
+ generateServiceAttributes(es, ps);
+ }
+ }
+ else if (e.getKey().type == Service.FILE_OPERATION) {
+ hasFileService = true;
+ ps.println("\tfilesystem {");
+ ps.println("\t\ttype: \"" + provider + "\"");
+ if (e.getValue().getServiceContact() != null) {
+ ps.println("\t\tURL: \"" + e.getValue().getServiceContact().getContact() + "\"");
+ }
+ ps.println("\t}");
+ }
+ }
+ if (!hasFileService) {
+ ps.println("\tstaging: \"provider\"");
+ }
+ if (bc.hasProperty("workdir")) {
+ ps.println("\tworkDirectory: \"" + bc.getProperty("workdir") + "\"");
+ }
+ if (bc.hasProperty("scratch")) {
+ ps.println("\tscratch: \"" + bc.getProperty("scratch") + "\"");
+ }
+ generateSiteAttributes(bc.getProperties(), ps);
+ if (tc != null) {
+ generateAppInfo(bc.getName(), tc, hasCoasterService, 1, ps);
+ }
+ ps.println("}\n");
+ }
+ }
+ if (tc != null) {
+ generateAppInfo("*", tc, false, 0, ps);
+ }
+ }
+
+ private void generateAppInfo(String host, TransformationCatalog tc, boolean hasCoasterService, int level, PrintStream ps) {
+ try {
+ for (TCEntry e : tc.getTC()) {
+ if (e.getResourceId().equals(host)) {
+ if (e.getLogicalName().equals("*")) {
+ tabs(level, ps);
+ ps.println("app.ALL {");
+ }
+ else {
+ tabs(level, ps);
+ ps.println("app." + e.getLogicalName() + " {");
+ }
+ tabs(level + 1, ps);
+ ps.println("executable: " + e.getPhysicalTransformation());
+ if (e.getProfiles() != null) {
+ generateAppProfiles(e, hasCoasterService, level, ps);
+ }
+ tabs(level, ps);
+ ps.println("}\n");
+ }
+ }
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Failed to get TC data", e);
+ }
+ }
+
+ private void generateAppProfiles(TCEntry e, boolean hasCoasterService, int level, PrintStream ps) {
+ StringBuilder options = new StringBuilder();
+ for (Profile p : e.getProfiles()) {
+ if (p.getProfileKey().startsWith("env.")) {
+ tabs(level, ps);
+ ps.println(p.getProfileKey() + ": \"" + p.getProfileValue() + "\"");
+ }
+ else if (p.getProfileKey().equals("maxwalltime")) {
+ tabs(level, ps);
+ ps.println("maxWallTime: \"" + p.getProfileValue() + "\"");
+ }
+ else if (p.getProfileKey().equals("queue")) {
+ if (!hasCoasterService) {
+ tabs(level, ps);
+ ps.println("jobQueue: \"" + p.getProfileValue() + "\"");
+ }
+ }
+ else if (p.getProfileKey().equals("project")) {
+ if (!hasCoasterService) {
+ tabs(level, ps);
+ ps.println("jobProject: \"" + p.getProfileValue() + "\"");
+ }
+ }
+ else {
+ if (options.length() != 0) {
+ options.append(", ");
+ }
+ else {
+ options.append("{");
+ }
+ options.append(p.getProfileKey() + ": \"" + p.getProfileValue() + "\"");
+ }
+ }
+ if (options.length() != 0) {
+ options.append("}");
+ tabs(level, ps);
+ ps.println("options: " + options.toString());
+ }
+ ps.println("\t}");
+ }
+
+ private void tabs(int count, PrintStream ps) {
+ for (int i = 0; i < count; i++) {
+ ps.print('\t');
+ }
+ }
+
+ private void generateSiteAttributes(Map<String, Object> properties, PrintStream ps) {
+ if (properties.containsKey("sysinfo")) {
+ ps.println("\tOS: \"" + properties.get("sysinfo") + "\"");
+ }
+ if (properties.containsKey("delayBase")) {
+ ps.println("\tdelayBase: " + properties.get("delayBase"));
+ }
+ if (properties.containsKey("initialScore")) {
+ double jobThrottle = 2;
+ if (properties.containsKey("jobThrottle")) {
+ jobThrottle = TypeUtil.toDouble(properties.get("jobThrottle"));
+ }
+ double initialScore = TypeUtil.toDouble(properties.get("initialScore"));
+ ps.println("\tmaxParallelTasks: " + (int) (jobThrottle * WeightedHost.T + 1));
+ ps.println("\tinitialParallelTasks: " + (int) (jobThrottle * WeightedHost.computeTScore(initialScore) + 1));
+ }
+ if (properties.containsKey("maxSubmitRate")) {
+ ps.println("\tmaxSubmitRate: " + properties.get("maxSubmitRate"));
+ }
+ }
+
+ private enum AttrType {
+ INT, FLOAT, STRING, BOOLEAN, TIME_FROM_SECONDS;
+ }
+
+ private static class Attr {
+ public Attr(String k, AttrType t) {
+ this.name = k;
+ this.type = t;
+ }
+ public AttrType type;
+ public String name;
+ }
+
+ private static final Set<String> removedAttrs;
+ private static final Map<String, Attr> coasterAttrs;
+ private static final Map<String, Attr> serviceAttrs;
+ private static final Map<String, Attr> props;
+ private static List<String> propOrder;
+
+ static {
+ removedAttrs = new HashSet<String>();
+ removedAttrs.add("pgraph");
+ removedAttrs.add("pgraph.graph.options");
+ removedAttrs.add("pgraph.node.options");
+ removedAttrs.add("clustering.enabled");
+ removedAttrs.add("clustering.queue.delay");
+ removedAttrs.add("clustering.min.time");
+ removedAttrs.add("kickstart.enabled");
+ removedAttrs.add("kickstart.always.transfer");
+ removedAttrs.add("sites.file");
+ removedAttrs.add("tc.file");
+
+ coasterAttrs = new HashMap<String, Attr>();
+ attr(coasterAttrs, "queue", "jobQueue", AttrType.STRING);
+ attr(coasterAttrs, "project", "jobProject", AttrType.STRING);
+ attr(coasterAttrs, "maxtime", "jobMaxTime", AttrType.TIME_FROM_SECONDS);
+ attr(coasterAttrs, "reserve", AttrType.STRING);
+ attr(coasterAttrs, "lowOverallocation", AttrType.INT);
+ attr(coasterAttrs, "highOverallocation", AttrType.INT);
+ attr(coasterAttrs, "slots", "maxJobs", AttrType.INT);
+ attr(coasterAttrs, "maxNodes", "maxNodesPerJob", AttrType.INT);
+ attr(coasterAttrs, "overallocationDecayFactor", AttrType.FLOAT);
+ attr(coasterAttrs, "internalHostname", AttrType.STRING);
+ attr(coasterAttrs, "allocationStepSize", AttrType.FLOAT);
+ attr(coasterAttrs, "nodeGranularity", AttrType.INT);
+ attr(coasterAttrs, "remoteMonitorEnabled", AttrType.BOOLEAN);
+ attr(coasterAttrs, "userHomeOverride", AttrType.STRING);
+ attr(coasterAttrs, "workerLoggingLevel", AttrType.STRING);
+ attr(coasterAttrs, "workerLoggingDirectory", AttrType.STRING);
+ attr(coasterAttrs, "workerManager", AttrType.STRING);
+ attr(coasterAttrs, "softImage", AttrType.STRING);
+ attr(coasterAttrs, "jobsPerNode", "tasksPerNode", AttrType.INT);
+
+ serviceAttrs = new HashMap<String, Attr>();
+ attr(serviceAttrs, "queue", "jobQueue", AttrType.STRING);
+ attr(serviceAttrs, "project", "jobProject", AttrType.STRING);
+ attr(serviceAttrs, "maxtime", "jobMaxTime", AttrType.STRING);
+ attr(serviceAttrs, "jobType", "jobType", AttrType.STRING);
+
+ props = new HashMap<String, Attr>();
+ propOrder = new ArrayList<String>();
+ prop("hostname", "hostName", AttrType.STRING);
+ prop("tcp.port.range", "TCPPortRange", AttrType.STRING);
+ prop("lazy.errors", "lazyErrors", AttrType.BOOLEAN);
+ prop("execution.retries", "executionRetries", AttrType.INT);
+ prop("caching.algorithm", "cachingAlgorithm", AttrType.STRING);
+ prop("throttle.submit", "jobSubmitThrottle", AttrType.INT);
+ prop("throttle.host.submit", "hostJobSubmitThrottle", AttrType.INT);
+ prop("throttle.transfers", "fileTransfersThrottle", AttrType.INT);
+ prop("throttle.file.operations", "fileOperationsThrottle", AttrType.INT);
+ prop("throttle.score.job.factor", "siteScoreThrottlingFactor", AttrType.INT);
+ prop("sitedir.keep", "keepSiteDir", AttrType.BOOLEAN);
+ prop("provenance.log", "logProvenance", AttrType.BOOLEAN);
+ prop("replication.enabled", "replicationEnabled", AttrType.BOOLEAN);
+ prop("replication.min.queue.time", "replicationMinQueueTime", AttrType.INT);
+ prop("replication.limit", "replicationLimit", AttrType.INT);
+ prop("status.mode", "statusMode", AttrType.STRING);
+ prop("wrapper.parameter.mode", "wrapperParameterMode", AttrType.STRING);
+ prop("wrapper.invocation.mode", "wrapperInvocationMode", AttrType.STRING);
+ prop("cdm.broadcast.mode", "CDMBroadcastMode", AttrType.STRING);
+ prop("provider.staging.pin.swiftfiles", "providerStagingPinSwiftFiles", AttrType.BOOLEAN);
+ prop("ticker.date.format", "tickerDateFormat", AttrType.STRING);
+ prop("ticker.prefix", "tickerPrefix", AttrType.STRING);
+ prop("ticker.enabled", "tickerEnabled", AttrType.BOOLEAN);
+ prop("file.gc.enabled", "fileGCEnabled", AttrType.BOOLEAN);
+ prop("mapping.checker", "mappingCheckerEnabled", AttrType.BOOLEAN);
+ prop("wrapperlog.always.transfer", "alwaysTransferWrapperLog", AttrType.BOOLEAN);
+ prop("tracing.enabled", "tracingEnabled", AttrType.BOOLEAN);
+ prop("wrapper.staging.local.server", "wrapperStagingLocalServer", AttrType.STRING);
+ prop("foreach.max.threads", "maxForeachThreads", AttrType.INT);
+ }
+
+
+ private static void attr(Map<String, Attr> m, String k, AttrType t) {
+ m.put("globus:" + k.toLowerCase(), new Attr(k, t));
+ }
+
+ private static void attr(Map<String, Attr> m, String k, String v, AttrType t) {
+ m.put("globus:" + k.toLowerCase(), new Attr(v, t));
+ }
+
+ private static void prop(String k, AttrType t) {
+ if (!SwiftConfig.SCHEMA.isNameValid(k)) {
+ throw new Error("Invalid prop: '" + k + "'");
+ }
+ props.put(k, new Attr(k, t));
+ propOrder.add(k);
+ }
+
+ private static void prop(String k, String v, AttrType t) {
+ if (!SwiftConfig.SCHEMA.isNameValid(v)) {
+ throw new Error("Invalid prop: '" + v + "'");
+ }
+ props.put(k, new Attr(v, t));
+ propOrder.add(k);
+ }
+
+ private void generateCoasterServiceAttributes(BoundContact bc, PrintStream ps) {
+ for (String attr : bc.getProperties().keySet()) {
+ Attr a = coasterAttrs.get(attr.toLowerCase());
+ if (a != null) {
+ ps.print("\t\t" + a.name + ": ");
+ printValue(a.type, bc.getProperty(attr), ps);
+ }
+ else {
+ if (attr.startsWith("globus.")) {
+ throw new RuntimeException("Unrecognize profile: '" + attr + "'");
+ }
+ }
+ }
+ }
+
+ private void printValue(AttrType type, Object value, PrintStream ps) {
+ switch (type) {
+ case INT:
+ case FLOAT:
+ case BOOLEAN:
+ ps.println(value);
+ break;
+ case STRING:
+ ps.println("\"" + value + "\"");
+ break;
+ case TIME_FROM_SECONDS:
+ ps.println("\"" + WallTime.format("hms", TypeUtil.toInt(value)) + "\"");
+ break;
+ }
+ }
+
+ private void generateServiceAttributes(Service s, PrintStream ps) {
+ for (String attr : s.getAttributeNames()) {
+ Attr a = serviceAttrs.get(attr);
+ if (a != null) {
+ ps.print("\t\t" + a.name + ": ");
+ switch (a.type) {
+ case INT:
+ case FLOAT:
+ case BOOLEAN:
+ ps.println(s.getAttribute(attr));
+ break;
+ case STRING:
+ ps.println("\"" + s.getAttribute(attr) + "\"");
+ break;
+ }
+ }
+ else {
+ if (attr.startsWith("globus.") && coasterAttrs.containsKey(attr)) {
+ throw new RuntimeException("Unrecognize profile: '" + attr + "'");
+ }
+ }
+ }
+ }
+
+ private ContactSet buildResources(Document doc) {
+ Node root = getRoot(doc);
+
+ if (root.getLocalName().equals("config")) {
+ return parse(root, Version.V1);
+ }
+ else if (root.getLocalName().equals("sites")) {
+ return parse(root, Version.V1);
+ }
+ else {
+ throw new IllegalArgumentException("Illegal sites file root node: " + root.getLocalName());
+ }
+ }
+
+
+ private ContactSet parse(Node config, Version v) {
+ ContactSet cs = new ContactSet();
+ NodeList pools = config.getChildNodes();
+ for (int i = 0; i < pools.getLength(); i++) {
+ Node n = pools.item(i);
+ if (n.getNodeType() == Node.ELEMENT_NODE) {
+ try {
+ BoundContact bc = pool(n, v);
+ if (bc != null) {
+ cs.addContact(bc);
+ }
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Invalid site entry '" + poolName(n, v) + "': ", e);
+ }
+ }
+ }
+ return cs;
+ }
+
+ private String poolName(Node site, Version v) {
+ if (site.getLocalName().equals("pool")) {
+ return attr(site, "handle");
+ }
+ else if (site.getLocalName().equals("site")) {
+ return attr(site, "name");
+ }
+ else {
+ throw new IllegalArgumentException("Invalid node: " + site.getLocalName());
+ }
+ }
+
+ private Node getRoot(Document doc) {
+ NodeList l = doc.getChildNodes();
+ for (int i = 0; i < l.getLength(); i++) {
+ if (l.item(i).getNodeType() == Node.ELEMENT_NODE) {
+ return l.item(i);
+ }
+ }
+ throw new IllegalArgumentException("Missing root element");
+ }
+
+ private BoundContact pool(Node n, Version v) throws InvalidProviderException, ProviderMethodException {
+ if (n.getNodeType() != Node.ELEMENT_NODE) {
+ return null;
+ }
+ String name = poolName(n, v);
+ BoundContact bc = new BoundContact(name);
+
+ String sysinfo = attr(n, "sysinfo", null);
+ if (sysinfo != null) {
+ bc.setProperty("sysinfo", sysinfo);
+ }
+
+ NodeList cs = n.getChildNodes();
+
+ for (int i = 0; i < cs.getLength(); i++) {
+ Node c = cs.item(i);
+ if (c.getNodeType() != Node.ELEMENT_NODE) {
+ continue;
+ }
+ String ctype = c.getNodeName();
+
+ if (v == Version.V1 && ctype.equals("gridftp")) {
+ bc.addService(gridftp(c));
+ }
+ else if (v == Version.V1 && ctype.equals("jobmanager")) {
+ bc.addService(jobmanager(c));
+ }
+ else if (ctype.equals("execution")) {
+ bc.addService(execution(c));
+ }
+ else if (ctype.equals("filesystem")) {
+ bc.addService(filesystem(c));
+ }
+ else if (ctype.equals("workdirectory")) {
+ bc.setProperty("workdir", text(c));
+ }
+ else if (ctype.equals("scratch")) {
+ bc.setProperty("scratch", text(c));
+ }
+ else if (ctype.equals("env")) {
+ env(bc, c);
+ }
+ else if (ctype.equals("profile")) {
+ profile(bc, c);
+ }
+ else {
+ throw new IllegalArgumentException("Unknown node type: " + ctype);
+ }
+ }
+ return bc;
+ }
+
+ private Service jobmanager(Node n) throws InvalidProviderException, ProviderMethodException {
+ String provider;
+ String url = attr(n, "url");
+ String major = attr(n, "major");
+ if (url.equals("local://localhost")) {
+ provider = "local";
+ }
+ else if (url.equals("pbs://localhost")) {
+ provider = "pbs";
+ }
+ else if ("2".equals(major)) {
+ provider = "gt2";
+ }
+ else if ("4".equals(major)) {
+ provider = "gt4";
+ }
+ else {
+ throw new IllegalArgumentException("Unknown job manager version: " + major + ", url = '" + url + "'");
+ }
+
+ ServiceContact contact = new ServiceContactImpl(url);
+ return new ServiceImpl(provider, Service.EXECUTION,
+ contact, AbstractionFactory.newSecurityContext(provider, contact));
+ }
+
+ private Service gridftp(Node n) throws InvalidProviderException, ProviderMethodException {
+ String url = attr(n, "url");
+ if (url.equals("local://localhost")) {
+ return new ServiceImpl("local", Service.FILE_OPERATION, new ServiceContactImpl("localhost"), null);
+ }
+ else {
+ ServiceContact contact = new ServiceContactImpl(url);
+ return new ServiceImpl("gsiftp", Service.FILE_OPERATION,
+ contact, AbstractionFactory.newSecurityContext("gsiftp", contact));
+ }
+ }
+
+ private Service execution(Node n) throws InvalidProviderException, ProviderMethodException {
+ String provider = attr(n, "provider");
+ String url = attr(n, "url", null);
+ String jobManager = attr(n, "jobManager", null);
+ if (jobManager == null) {
+ jobManager = attr(n, "jobmanager", null);
+ }
+
+ ExecutionService s = new ExecutionServiceImpl();
+ s.setProvider(provider);
+ ServiceContact contact = null;
+ if (url != null) {
+ contact = new ServiceContactImpl(url);
+ s.setServiceContact(contact);
+ s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact));
+ }
+
+ if (jobManager != null) {
+ s.setJobManager(jobManager);
+ }
+
+ return s;
+ }
+
+ private Service filesystem(Node n) throws InvalidProviderException, ProviderMethodException {
+ String provider = attr(n, "provider");
+ String url = attr(n, "url", null);
+
+ Service s = new ServiceImpl();
+ s.setType(Service.FILE_OPERATION);
+ s.setProvider(provider);
+
+ ServiceContact contact = null;
+ if (url != null) {
+ contact = new ServiceContactImpl(url);
+ s.setServiceContact(contact);
+ s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact));
+ }
+
+ return s;
+ }
+
+ private void env(BoundContact bc, Node n) {
+ String key = attr(n, "key");
+ String value = text(n);
+
+ bc.setProperty("env:" + key, value);
+ }
+
+ private void profile(BoundContact bc, Node n) {
+ String ns = attr(n, "namespace");
+ String key = attr(n, "key");
+ String value = text(n);
+
+ if (value == null) {
+ throw new IllegalArgumentException("No value for profile " + ns + ":" + key);
+ }
+ if (ns.equals("karajan")) {
+ bc.setProperty(key, value);
+ }
+ else {
+ bc.setProperty(ns + ":" + key, value);
+ }
+ }
+
+ private String text(Node n) {
+ if (n.getFirstChild() != null) {
+ return n.getFirstChild().getNodeValue();
+ }
+ else {
+ return null;
+ }
+ }
+
+ private String attr(Node n, String name) {
+ NamedNodeMap attrs = n.getAttributes();
+ if (attrs != null) {
+ Node attr = attrs.getNamedItem(name);
+ if (attr == null) {
+ throw new IllegalArgumentException("Missing " + name);
+ }
+ else {
+ return attr.getNodeValue();
+ }
+ }
+ else {
+ throw new IllegalArgumentException("Missing " + name);
+ }
+ }
+
+ private String attr(Node n, String name, String defVal) {
+ NamedNodeMap attrs = n.getAttributes();
+ if (attrs != null) {
+ Node attr = attrs.getNamedItem(name);
+ if (attr == null) {
+ return defVal;
+ }
+ else {
+ return attr.getNodeValue();
+ }
+ }
+ else {
+ return defVal;
+ }
+ }
+
+ public static void main(String[] args) {
+ ArgumentParser ap = new ArgumentParser();
+ ap.setExecutableName("swift-convert-config");
+ ap.addFlag("help", "Displays usage information");
+ ap.addOption("sites.file", "Specifies a sites file to convert",
+ "file", ArgumentParser.OPTIONAL);
+ ap.addOption("tc.file", "Specifies a tc.data file to convert",
+ "file", ArgumentParser.OPTIONAL);
+ ap.addOption("config", "Specifies an old Swift configuration file to convert",
+ "file", ArgumentParser.OPTIONAL);
+ ap.addOption("out", "Indicates that the output should go to a file instead of standard output",
+ "file", ArgumentParser.OPTIONAL);
+
+ try {
+ ap.parse(args);
+ }
+ catch (ArgumentParserException e) {
+ System.out.println("Error parsing command line arguments: " + e.getMessage());
+ usage(ap);
+ System.exit(1);
+ }
+ if (ap.isPresent("help")) {
+ usage(ap);
+ System.exit(0);
+ }
+ ConvertConfig cc = new ConvertConfig();
+ try {
+ PrintStream ps = System.out;
+ if (ap.isPresent("out")) {
+ ps = new PrintStream(new FileOutputStream(ap.getStringValue("out")));
+ }
+ cc.run(ap.getStringValue("sites.file", null),
+ ap.getStringValue("tc.file", null), ap.getStringValue("config", null), System.out);
+ System.exit(0);
+ }
+ catch (Exception e) {
+ System.out.println("Failed to convert configuration: " + e.getMessage());
+ e.printStackTrace();
+ System.exit(2);
+ }
+ }
+
+ private static void usage(ArgumentParser ap) {
+ System.out.println(
+ "Converts an existing set of sites.xml, tc.data, and swift.properties \n" +
+ "files (at least one of which needs to be specified) to a unified Swift\n" +
+ "configuration file.\n");
+ ap.usage();
+ }
+}
Added: trunk/src/org/griphyn/vdl/util/SwiftConfig.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/SwiftConfig.java (rev 0)
+++ trunk/src/org/griphyn/vdl/util/SwiftConfig.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -0,0 +1,632 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 5, 2014
+ */
+package org.griphyn.vdl.util;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.log4j.Logger;
+import org.globus.cog.abstraction.impl.common.AbstractionFactory;
+import org.globus.cog.abstraction.impl.common.ProviderMethodException;
+import org.globus.cog.abstraction.impl.common.task.ExecutionServiceImpl;
+import org.globus.cog.abstraction.impl.common.task.InvalidProviderException;
+import org.globus.cog.abstraction.impl.common.task.ServiceContactImpl;
+import org.globus.cog.abstraction.impl.common.task.ServiceImpl;
+import org.globus.cog.abstraction.interfaces.ExecutionService;
+import org.globus.cog.abstraction.interfaces.Service;
+import org.globus.cog.abstraction.interfaces.ServiceContact;
+import org.globus.cog.karajan.util.BoundContact;
+import org.globus.swift.catalog.site.Application;
+import org.globus.swift.catalog.site.SwiftContact;
+import org.globus.swift.catalog.site.SwiftContactSet;
+import org.griphyn.vdl.util.ConfigTree.Node;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigIncludeContext;
+import com.typesafe.config.ConfigIncluder;
+import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigParseOptions;
+import com.typesafe.config.ConfigSyntax;
+import com.typesafe.config.ConfigValue;
+import com.typesafe.config.ConfigValueType;
+
+public class SwiftConfig implements Cloneable {
+ public static final Logger logger = Logger.getLogger(SwiftConfig.class);
+
+ public static final boolean CHECK_DYNAMIC_NAMES = true;
+ public static final List<String> DEFAULT_LOCATIONS;
+
+ public enum Key {
+ DM_CHECKER("mappingCheckerEnabled"),
+ PROVENANCE_LOG("logProvenance"),
+ FILE_GC_ENABLED("fileGCEnabled"),
+ TICKER_ENABLED("tickerEnabled"),
+ TICKER_DATE_FORMAT("tickerDateFormat"),
+ TICKER_PREFIX("tickerPrefix"),
+ TRACING_ENABLED("tracingEnabled"),
+ FOREACH_MAX_THREADS("maxForeachThreads"),
+ CACHING_ALGORITHM("cachingAlgorithm"),
+ REPLICATION_ENABLED("replicationEnabled"),
+ WRAPPER_STAGING_LOCAL_SERVER("wrapperStagingLocalServer"),
+ REPLICATION_MIN_QUEUE_TIME("replicationMinQueueTime"),
+ REPLICATION_LIMIT("replicationLimit"),
+ WRAPPER_INVOCATION_MODE("wrapperInvocationMode");
+
+ public String propName;
+ private Key(String propName) {
+ this.propName = propName;
+ }
+ }
+
+ public static SwiftConfigSchema SCHEMA;
+
+ static {
+ SCHEMA = new SwiftConfigSchema();
+ DEFAULT_LOCATIONS = new ArrayList<String>();
+ if (System.getenv("SWIFT_SITE_CONF") != null) {
+ DEFAULT_LOCATIONS.add(System.getenv("SWIFT_SITE_CONF"));
+ }
+ String swiftHome = System.getProperty("swift.home");
+ if (swiftHome == null) {
+ swiftHome = System.getenv("SWIFT_HOME");
+ if (swiftHome == null) {
+ throw new IllegalStateException("SWIFT_HOME is not set");
+ }
+ }
+
+ DEFAULT_LOCATIONS.add(swiftHome + File.separator +
+ "etc" + File.separator + "swift.conf");
+
+ // check keys against schema
+
+ for (Key k : Key.values()) {
+ if (!SCHEMA.isNameValid(k.propName)) {
+ throw new IllegalArgumentException("Invalid property name for config key '" + k + "': " + k.propName);
+ }
+ }
+ }
+
+ private static class KVPair {
+ public final String key;
+ public final String value;
+
+ public KVPair(String key, String value) {
+ this.key = key;
+ this.value = value;
+ }
+ }
+
+ private static class IncluderWrapper implements ConfigIncluder {
+ private final ConfigIncluder d;
+
+ public IncluderWrapper(ConfigIncluder d) {
+ this.d = d;
+ }
+
+ @Override
+ public ConfigIncluder withFallback(ConfigIncluder fallback) {
+ return this;
+ }
+
+ @Override
+ public ConfigObject include(ConfigIncludeContext context, String what) {
+ int b = what.indexOf("${");
+ while (b != -1) {
+ int e = what.indexOf("}", b);
+ String var = what.substring(b + 2, e);
+ what = what.substring(0, b) + resolve(var) + what.substring(e + 1);
+ b = what.indexOf("${");
+ }
+ return ConfigFactory.parseFile(new File(what)).root();
+ }
+
+ private String resolve(String var) {
+ String v = null;
+ if (var.startsWith("env.")) {
+ v = System.getenv(var.substring(4));
+ }
+ else {
+ v = System.getProperty(var);
+ }
+ if (v == null) {
+ throw new IllegalArgumentException("No such system property or environment variable: '" + var + "'");
+ }
+ return v;
+ }
+ }
+
+ public static SwiftConfig load(String fileName) {
+ return load(fileName, null);
+ }
+
+ public static SwiftConfig load(String fileName, Map<String, Object> override) {
+ logger.info("Loading swift configuration file: " + fileName);
+ ConfigParseOptions opt = ConfigParseOptions.defaults();
+ opt = opt.setIncluder(new IncluderWrapper(opt.getIncluder())).
+ setSyntax(ConfigSyntax.CONF).setAllowMissing(false);
+ Config conf = ConfigFactory.parseFile(new File(fileName), opt);
+ Config oconf = ConfigFactory.parseMap(override, "<command line>");
+ conf = oconf.withFallback(conf);
+ conf = conf.resolveWith(getSubstitutions());
+ ConfigTree<Object> out = SCHEMA.validate(conf);
+ SwiftConfig sc = new SwiftConfig();
+ sc.setFileName(fileName);
+ sc.build(out);
+ return sc;
+ }
+
+ private static Config getSubstitutions() {
+ Map<String, Object> m = new HashMap<String, Object>();
+
+ for (Map.Entry<String, String> e : System.getenv().entrySet()) {
+ m.put("env." + e.getKey(), e.getValue());
+ }
+
+ return ConfigFactory.parseMap(m).withFallback(ConfigFactory.parseProperties(System.getProperties()));
+ }
+
+ public static SwiftConfig load() {
+ for (String loc : DEFAULT_LOCATIONS) {
+ if (new File(loc).exists()) {
+ return load(loc);
+ }
+ }
+ throw new IllegalStateException("Could not find swift configuration file");
+ }
+
+ private static SwiftConfig _default;
+
+ public synchronized static SwiftConfig getDefault() {
+ if (_default == null) {
+ _default = load();
+ }
+ return _default;
+ }
+
+ public synchronized static void setDefault(SwiftConfig conf) {
+ _default = conf;
+ }
+
+ private SwiftContactSet definedSites;
+ private SwiftContactSet sites;
+ private ConfigTree<Object> tree;
+ private Map<String, Object> flat;
+ private String fileName;
+
+ public SwiftConfig() {
+ definedSites = new SwiftContactSet();
+ sites = new SwiftContactSet();
+ flat = new HashMap<String, Object>();
+ }
+
+ public SwiftContactSet getSites() {
+ return sites;
+ }
+
+ public SwiftContactSet getDefinedSites() {
+ return definedSites;
+ }
+
+ public Collection<String> getDefinedSiteNames() {
+ Set<String> s = new TreeSet<String>();
+ for (BoundContact bc : definedSites.getContacts()) {
+ s.add(bc.getName());
+ }
+ return s;
+ }
+
+ public void setProperty(String key, Object value) {
+ flat.put(key, value);
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void build(ConfigTree<Object> tree) {
+ this.tree = tree;
+ List<String> sites = null;
+ for (Map.Entry<String, ConfigTree.Node<Object>> e : tree.entrySet()) {
+ if (e.getKey().equals("site")) {
+ for (Map.Entry<String, ConfigTree.Node<Object>> f : e.getValue().entrySet()) {
+ site(definedSites, f.getKey(), f.getValue());
+ }
+ }
+ else if (e.getKey().equals("sites")) {
+ sites = (List<String>) getObject(e.getValue());
+ }
+ else if (e.getKey().equals("app")) {
+ SwiftContact dummy = new SwiftContact();
+ apps(dummy, e.getValue());
+ for (Application app : dummy.getApplications()) {
+ definedSites.addApplication(app);
+ }
+ }
+ }
+ if (sites == null || sites.isEmpty()) {
+ throw new RuntimeException("No sites enabled");
+ }
+ for (String siteName : sites) {
+ this.sites.addContact((SwiftContact) definedSites.getContact(siteName));
+ }
+ this.sites.getApplications().putAll(definedSites.getApplications());
+
+ for (String leaf : tree.getLeafPaths()) {
+ flat.put(leaf, tree.get(leaf));
+ }
+ }
+
+ private void apps(SwiftContact sc, ConfigTree.Node<Object> n) {
+ /*
+ * app."*" {
+ * ...
+ * }
+ */
+
+ for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ String k = e.getKey();
+ ConfigTree.Node<Object> c = e.getValue();
+
+ if (e.getKey().equals("ALL")) {
+ sc.addApplication(app("*", e.getValue()));
+ }
+ else {
+ sc.addApplication(app(removeQuotes(e.getKey()), e.getValue()));
+ }
+ }
+
+ Application all = sc.getApplication("*");
+ if (all != null) {
+ mergeEnvsToApps(sc, all.getEnv());
+ mergePropsToApps(sc, all.getProperties());
+ }
+ }
+
+ private String removeQuotes(String key) {
+ if (key.startsWith("\"") && key.endsWith("\"")) {
+ return key.substring(1, key.length() - 2);
+ }
+ else {
+ return key;
+ }
+ }
+
+ private void mergeEnvsToApps(SwiftContact bc, Map<String, String> envs) {
+ for (Application app : bc.getApplications()) {
+ for (Map.Entry<String, String> e : envs.entrySet()) {
+ if (!app.getEnv().containsKey(e.getKey())) {
+ // only merge if app does not override
+ app.setEnv(e.getKey(), e.getValue());
+ }
+ }
+ }
+ }
+
+ private void mergePropsToApps(SwiftContact bc, Map<String, Object> props) {
+ for (Application app : bc.getApplications()) {
+ for (Map.Entry<String, Object> e : props.entrySet()) {
+ if (!app.getProperties().containsKey(e.getKey())) {
+ app.addProperty(e.getKey(), e.getValue());
+ }
+ }
+ }
+ }
+
+ private Application app(String name, ConfigTree.Node<Object> n) {
+ /*
+ * app."*" {
+ * executable: "?String"
+ *
+ * options: "?Object"
+ * jobType: "?String", queue: "?String", project: "?String"
+ * maxWallTime: "?Time"
+ *
+ * env."*": "String"
+ * }
+ */
+
+ Application app = new Application();
+ app.setName(name);
+
+ for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ String k = e.getKey();
+ ConfigTree.Node<Object> c = e.getValue();
+
+ if (k.equals("executable")) {
+ app.setExecutable(getString(c));
+ }
+ else if (k.equals("options")) {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> opt = (Map<String, Object>) getObject(c, "options");
+ for (Map.Entry<String, Object> f : opt.entrySet()) {
+ app.addProperty(f.getKey(), f.getValue());
+ }
+ }
+ else if (k.equals("jobQueue")) {
+ app.addProperty("queue", getString(c));
+ }
+ else if (k.equals("jobProject")) {
+ app.addProperty("project", getString(c));
+ }
+ else if (k.equals("env")) {
+ List<KVPair> envs = envs(c);
+ for (KVPair env : envs) {
+ app.setEnv(env.key, env.value);
+ }
+ }
+ else {
+ app.addProperty(k, getString(c));
+ }
+ }
+
+ return app;
+ }
+
+
+ private List<KVPair> envs(Node<Object> n) {
+ List<KVPair> l = new ArrayList<KVPair>();
+ for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ l.add(new KVPair(e.getKey(), getString(e.getValue())));
+ }
+ return l;
+ }
+
+ private void site(SwiftContactSet sites, String name, ConfigTree.Node<Object> n) {
+ try {
+ SwiftContact sc = new SwiftContact(name);
+
+ if (n.hasKey("OS")) {
+ sc.setProperty("sysinfo", getString(n, "OS"));
+ }
+
+
+ for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ String ctype = e.getKey();
+ ConfigTree.Node<Object> c = e.getValue();
+
+ if (ctype.equals("execution")) {
+ sc.addService(execution(c));
+ }
+ else if (ctype.equals("filesystem")) {
+ sc.addService(filesystem(c));
+ }
+ else if (ctype.equals("workDirectory")) {
+ sc.setProperty("workdir", getString(c));
+ }
+ else if (ctype.equals("scratch")) {
+ sc.setProperty("scratch", getString(c));
+ }
+ else if (ctype.equals("app")) {
+ apps(sc, c);
+ }
+ else if (ctype.equals("options")) {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> opt = (Map<String, Object>) getObject(c, "options");
+ for (Map.Entry<String, Object> f : opt.entrySet()) {
+ sc.setProperty(f.getKey(), f.getValue());
+ }
+ }
+ else {
+ sc.setProperty(ctype, getObject(c));
+ }
+ }
+ sites.addContact(sc);
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Invalid site entry '" + name + "': ", e);
+ }
+ }
+
+ private Service filesystem(Node<Object> c) throws InvalidProviderException, ProviderMethodException {
+ Service s = new ServiceImpl();
+ s.setType(Service.FILE_OPERATION);
+ service(c, s);
+ return s;
+ }
+
+ private Service execution(ConfigTree.Node<Object> n) throws InvalidProviderException, ProviderMethodException {
+ ExecutionService s = new ExecutionServiceImpl();
+ service(n, s);
+ return s;
+ }
+
+ private void service(Node<Object> n, Service s) throws InvalidProviderException, ProviderMethodException {
+ String provider = null;
+ String url = null;
+ for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ String k = e.getKey();
+ ConfigTree.Node<Object> c = e.getValue();
+
+ if (k.equals("type")) {
+ provider = getString(c);
+ }
+ else if (k.equals("URL")) {
+ url = getString(c);
+ }
+ else if (k.equals("jobManager")) {
+ ((ExecutionService) s).setJobManager(getString(c));
+ }
+ else if (k.equals("jobProject")) {
+ s.setAttribute("project", getObject(c));
+ }
+ else if (k.equals("maxJobs")) {
+ s.setAttribute("slots", getObject(c));
+ }
+ else if (k.equals("maxJobTime")) {
+ s.setAttribute("maxTime", getObject(c));
+ }
+ else if (k.equals("maxNodesPerJob")) {
+ s.setAttribute("maxNodes", getObject(c));
+ }
+ else if (k.equals("jobQueue")) {
+ s.setAttribute("queue", getObject(c));
+ }
+ else {
+ s.setAttribute(k, getObject(c));
+ }
+ }
+
+ s.setProvider(provider);
+ if (url != null) {
+ ServiceContact contact = new ServiceContactImpl(url);
+ s.setServiceContact(contact);
+ s.setSecurityContext(AbstractionFactory.newSecurityContext(provider, contact));
+ }
+ }
+
+ private String getString(Node<Object> c) {
+ return (String) c.get();
+ }
+
+ private Object getObject(ConfigTree.Node<Object> c) {
+ return c.get();
+ }
+
+ private String getString(ConfigTree.Node<Object> c, String key) {
+ return (String) c.get(key);
+ }
+
+ private Object getObject(ConfigTree.Node<Object> c, String key) {
+ return c.get(key);
+ }
+
+ private void checkType(String name, ConfigValue value, ConfigValueType type) {
+ if (!type.equals(value.valueType())) {
+ throw new SwiftConfigException(value.origin(),
+ "'" + name + "': wrong type (" + value.valueType() + "). Must be a " + type);
+ }
+ }
+
+ public Object clone() {
+ return this;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ SortedSet<String> s = new TreeSet<String>(flat.keySet());
+ for (String k : s) {
+ sb.append(k);
+ sb.append(": ");
+ Object o = flat.get(k);
+ if (o instanceof String) {
+ sb.append('"');
+ sb.append(o);
+ sb.append('"');
+ }
+ else {
+ sb.append(o);
+ }
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+
+ private void check(String name) {
+ if (CHECK_DYNAMIC_NAMES) {
+ if (!SCHEMA.isNameValid(name)) {
+ throw new IllegalArgumentException("Unknown property name: '" + name + "'");
+ }
+ }
+ }
+
+ private Object get(Key k) {
+ return flat.get(k.propName);
+ }
+
+ public Object getProperty(String name) {
+ check(name);
+ return flat.get(name);
+ }
+
+ public String getStringProperty(String name) {
+ check(name);
+ return (String) flat.get(name);
+ }
+
+ public boolean isFileGCEnabled() {
+ return (Boolean) get(Key.FILE_GC_ENABLED);
+ }
+
+ public boolean isTickerEnabled() {
+ return (Boolean) get(Key.TICKER_ENABLED);
+ }
+
+ public String getTickerDateFormat() {
+ return (String) get(Key.TICKER_DATE_FORMAT);
+ }
+
+ public String getTickerPrefix() {
+ return (String) get(Key.TICKER_PREFIX);
+ }
+
+ public boolean isTracingEnabled() {
+ return (Boolean) get(Key.TRACING_ENABLED);
+ }
+
+ public Object getProperty(String name, Object defVal) {
+ check(name);
+ Object v = flat.get(name);
+ if (v == null) {
+ return defVal;
+ }
+ else {
+ return v;
+ }
+ }
+
+ public int getForeachMaxThreads() {
+ return (Integer) get(Key.FOREACH_MAX_THREADS);
+ }
+
+ public String getCachingAlgorithm() {
+ return (String) get(Key.CACHING_ALGORITHM);
+ }
+
+ public boolean isReplicationEnabled() {
+ return (Boolean) get(Key.REPLICATION_ENABLED);
+ }
+
+ public String getWrapperStagingLocalServer() {
+ return (String) get(Key.WRAPPER_STAGING_LOCAL_SERVER);
+ }
+
+ public boolean isProvenanceEnabled() {
+ return (Boolean) get(Key.PROVENANCE_LOG);
+ }
+
+ public int getReplicationMinQueueTime() {
+ return (Integer) get(Key.REPLICATION_MIN_QUEUE_TIME);
+ }
+
+ public int getReplicationLimit() {
+ return (Integer) get(Key.REPLICATION_LIMIT);
+ }
+
+ public String getWrapperInvocationMode() {
+ return (String) get(Key.WRAPPER_INVOCATION_MODE);
+ }
+
+ public boolean isMappingCheckerEnabled() {
+ return (Boolean) get(Key.DM_CHECKER);
+ }
+}
Added: trunk/src/org/griphyn/vdl/util/SwiftConfigException.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/SwiftConfigException.java (rev 0)
+++ trunk/src/org/griphyn/vdl/util/SwiftConfigException.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -0,0 +1,20 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 5, 2014
+ */
+package org.griphyn.vdl.util;
+
+import com.typesafe.config.ConfigOrigin;
+
+public class SwiftConfigException extends RuntimeException {
+ private ConfigOrigin loc;
+
+ public SwiftConfigException(ConfigOrigin loc, String message) {
+ super((loc.filename() == null ? loc.description() : loc.filename() + ":" + loc.lineNumber()) + " " + message);
+ }
+}
Added: trunk/src/org/griphyn/vdl/util/SwiftConfigInfo.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/SwiftConfigInfo.java (rev 0)
+++ trunk/src/org/griphyn/vdl/util/SwiftConfigInfo.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -0,0 +1,71 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 8, 2014
+ */
+package org.griphyn.vdl.util;
+
+import java.util.Collection;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.globus.cog.util.ArgumentParser;
+import org.globus.cog.util.StringUtil;
+
+public class SwiftConfigInfo {
+ public static void main(String[] args) {
+ ArgumentParser ap = new ArgumentParser();
+ ap.setExecutableName("swift-config-info");
+ ap.addFlag("help", "Prints usage information");
+ ap.addFlag("list", "Lists all available configuration properties");
+ ap.addOption(ArgumentParser.DEFAULT, "A property to print information about",
+ "property", ArgumentParser.OPTIONAL);
+ try {
+ ap.parse(args);
+ if (ap.isPresent("help")) {
+ ap.usage();
+ System.exit(0);
+ }
+ if (ap.isPresent("list")) {
+ list();
+ }
+ if (ap.hasValue(ArgumentParser.DEFAULT)) {
+ info(ap.getStringValue(ArgumentParser.DEFAULT));
+ }
+ System.exit(0);
+ }
+ catch (Exception e) {
+ System.out.println("Error parsing command line: " + e.getMessage());
+ ap.usage();
+ System.exit(1);
+ }
+ }
+
+ private static void info(String prop) {
+ SwiftConfigSchema.Info info = SwiftConfig.SCHEMA.getPropertyDescriptions().get(prop);
+ if (info != null) {
+ if (info.doc == null) {
+ System.out.println(prop + ": no description available");
+ }
+ else {
+ System.out.println(prop + ":");
+ System.out.println(StringUtil.wordWrap(info.doc, 6, 65));
+ }
+ }
+ else {
+ System.out.println(prop + ": unknown property");
+ }
+ }
+
+ private static void list() {
+ Collection<String> names = SwiftConfig.SCHEMA.listProperties();
+ SortedSet<String> snames = new TreeSet<String>(names);
+ for (String name : snames) {
+ System.out.println(name.replace("\"*\"", "*"));
+ }
+ }
+}
Added: trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java (rev 0)
+++ trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -0,0 +1,288 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Jul 5, 2014
+ */
+package org.griphyn.vdl.util;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigOrigin;
+import com.typesafe.config.ConfigValue;
+
+public class SwiftConfigSchema {
+ private static final String STAR = "\"*\"";
+
+ public static class Info {
+ public ConfigPropertyType<?> type;
+ public Object value;
+ public boolean optional;
+ public String doc;
+ public ConfigOrigin loc;
+ }
+
+ private Config schema;
+ private ConfigTree<Info> info;
+ private Map<String, Info> descriptions;
+ private Set<String> validNames;
+
+ public SwiftConfigSchema() {
+ validNames = new HashSet<String>();
+ schema = ConfigFactory.parseResources("swift.conf.schema");
+ schema = schema.resolve();
+ if (schema.isEmpty()) {
+ throw new RuntimeException("Could not find swift.conf.schema");
+ }
+ info = new ConfigTree<Info>();
+ for (Map.Entry<String, ConfigValue> e : schema.entrySet()) {
+ String k = e.getKey();
+ String type = null;
+ Object defaultValue = null;
+ String doc = null;
+ ConfigOrigin loc = null;
+ if (k.endsWith(".\"_type\"")) {
+ type = schema.getString(k);
+ k = k.substring(0, k.lastIndexOf('.'));
+ loc = e.getValue().origin();
+ }
+ else if (k.indexOf(".\"_") == -1){
+ type = schema.getString(k);
+ loc = e.getValue().origin();
+ }
+ else if (k.endsWith(".\"_default\"")){
+ defaultValue = e.getValue().unwrapped();
+ k = k.substring(0, k.lastIndexOf('.'));
+ }
+ else if (k.endsWith(".\"_doc\"")){
+ doc = stripDoc((String) e.getValue().unwrapped());
+ k = k.substring(0, k.lastIndexOf('.'));
+ }
+ else if (k.indexOf(".\"_") != -1) {
+ continue;
+ }
+ Info i = info.get(k);
+ if (i == null) {
+ i = new Info();
+ info.put(k, i);
+ validNames.add(k);
+ }
+ if (type != null) {
+ if (type.startsWith("?")) {
+ i.optional = true;
+ type = type.substring(1);
+ }
+ i.type = getTypeInstance(type, e.getValue());
+ }
+ if (defaultValue != null) {
+ i.value = defaultValue;
+ }
+ if (doc != null) {
+ i.doc = doc;
+ }
+ if (loc != null) {
+ i.loc = loc;
+ }
+ }
+ }
+
+ private String stripDoc(String doc) {
+ return doc.replaceAll("[\\t\\n]+", "");
+ }
+
+ private ConfigPropertyType<?> getTypeInstance(String type, ConfigValue value) {
+ if (type.equals("String")) {
+ return ConfigPropertyType.STRING;
+ }
+ else if (type.equals("Boolean")) {
+ return ConfigPropertyType.BOOLEAN;
+ }
+ else if (type.equals("Int")) {
+ return ConfigPropertyType.INT;
+ }
+ else if (type.equals("Float")) {
+ return ConfigPropertyType.FLOAT;
+ }
+ else if (type.equals("StrictlyPositiveInt")) {
+ return ConfigPropertyType.STRICTLY_POSITIVE_INT;
+ }
+ else if (type.equals("Throttle")) {
+ return ConfigPropertyType.THROTTLE;
+ }
+ else if (type.equals("PortRange")) {
+ return ConfigPropertyType.PORT_RANGE;
+ }
+ else if (type.equals("PositiveInt")) {
+ return ConfigPropertyType.POSITIVE_INT;
+ }
+ else if (type.equals("PositiveFloat")) {
+ return ConfigPropertyType.POSITIVE_FLOAT;
+ }
+ else if (type.equals("Time")) {
+ return ConfigPropertyType.TIME;
+ }
+ else if (type.equals("Seconds")) {
+ return ConfigPropertyType.STRICTLY_POSITIVE_INT;
+ }
+ else if (type.equals("OS")) {
+ return ConfigPropertyType.OS;
+ }
+ else if (type.equals("StringList")) {
+ return ConfigPropertyType.STRING_LIST;
+ }
+ else if (type.equals("Object")) {
+ return ConfigPropertyType.OBJECT;
+ }
+ else if (type.startsWith("Choice[")) {
+ String values = type.substring("Choice[".length(), type.length() - 1);
+ return new ConfigPropertyType.Choices(values.split(",\\s*"));
+ }
+ else if (type.startsWith("Interval[")) {
+ String values = type.substring("Interval[".length(), type.length() - 1);
+ String[] s = values.split(",\\s*", 2);
+ return new ConfigPropertyType.Interval(Double.parseDouble(s[0]), Double.parseDouble(s[1]));
+ }
+ else {
+ throw new IllegalArgumentException(loc(value.origin()) + ": unknown property type: '" + type + "'");
+ }
+ }
+
+ private String loc(ConfigOrigin o) {
+ return o.filename() + ":" + o.lineNumber();
+ }
+
+ public ConfigTree<Object> validate(Config conf) {
+ ConfigTree<Object> validated = new ConfigTree<Object>();
+
+ // build a tree of the actual config so we can easily check them for missing properties
+ ConfigTree<Boolean> confTree = new ConfigTree<Boolean>();
+
+ // check things in config against schema
+ for (Map.Entry<String, ConfigValue> e : conf.entrySet()) {
+ String k = e.getKey();
+ confTree.put(k, Boolean.TRUE);
+ // check if the properties are defined in the schema
+ Info i = info.get(k, STAR);
+ if (i == null) {
+ i = info.get(k, STAR);
+ throw new SwiftConfigException(e.getValue().origin(), "unexpected property '" + k + "'");
+ }
+ // now check values
+ if (i.type == null) {
+ throw new IllegalStateException("Missing type for key " + k);
+ }
+ Object value = checkValue(k, e.getValue(), i.type);
+ validated.put(k, value);
+ }
+
+ // check for missing things
+ for (String key : info.getLeafPaths()) {
+ Info i = info.get(key);
+ String notFound = findMissing(key, confTree, i, validated);
+ if (notFound != null) {
+ findMissing(key, confTree, i, validated);
+ ConfigOrigin loc = info.get(key).loc;
+ throw new SwiftConfigException(conf.origin(), "missing property '" + notFound +
+ "' defined in " + loc.filename() + ":" + loc.lineNumber());
+ }
+ }
+ return validated;
+ }
+
+ private String findMissing(String key, ConfigTree<Boolean> confTree, Info i, ConfigTree<Object> validated) {
+ List<String> found = confTree.expandWildcards(key, STAR);
+ for (String f : found) {
+ if (!confTree.hasKey(f)) {
+ if (i.optional) {
+ if (i.value != null) {
+ validated.put(f, i.value);
+ }
+ }
+ else {
+ return f;
+ }
+ }
+ }
+ return null;
+ }
+
+ private void setValue(Map<String, Object> validated, Config conf, String k) {
+ if (!conf.hasPath(k)) {
+ String defKey = k + ".\"_default\"";
+ if (schema.hasPath(defKey)) {
+ validated.put(k, schema.getValue(defKey).unwrapped());
+ }
+ }
+ else {
+ validated.put(k, conf.getValue(k).unwrapped());
+ }
+ }
+
+ private Object checkValue(String k, ConfigValue value, ConfigPropertyType<?> t) {
+ Object v = value.unwrapped();
+ switch (value.valueType()) {
+ case STRING:
+ if (t.getBaseType() != ConfigPropertyType.STRING && t.getBaseType() != ConfigPropertyType.OBJECT) {
+ throw invalidValue(value, k, v, t.getBaseType());
+ }
+ return t.check(k, value.unwrapped(), null);
+ case NUMBER:
+ if (t.getBaseType() != ConfigPropertyType.INT && t.getBaseType() != ConfigPropertyType.FLOAT) {
+ throw invalidValue(value, k, v, t.getBaseType());
+ }
+ if (t.getBaseType() == ConfigPropertyType.INT) {
+ Number n = (Number) value.unwrapped();
+ if (n.intValue() != n.doubleValue()) {
+ throw invalidValue(value, k, v, t.getBaseType());
+ }
+ }
+ return t.check(k, v, null);
+ case BOOLEAN:
+ if (t.getBaseType() != ConfigPropertyType.BOOLEAN) {
+ throw invalidValue(value, k, v, t.getBaseType());
+ }
+ return value.unwrapped();
+ default:
+ return t.check(k, v, null);
+ }
+ }
+
+ private SwiftConfigException invalidValue(ConfigValue value, String k, Object v, ConfigPropertyType<?> t) {
+ return new SwiftConfigException(value.origin(), "invalid value '" + v + "' for property '" + k + "'. Expected a " + t);
+ }
+
+ public synchronized Map<String, Info> getPropertyDescriptions() {
+ if (descriptions == null) {
+ descriptions = new HashMap<String, Info>();
+ for (String s : info.getLeafPaths()) {
+ Info i = info.get(s);
+ if (i != null && i.doc != null) {
+ descriptions.put(s, i);
+ }
+ }
+ }
+ return descriptions;
+ }
+
+ public boolean propertyExists(String name) {
+ return info.get(name) != null;
+ }
+
+ public boolean isNameValid(String name) {
+ return validNames.contains(name);
+ }
+
+ public Collection<String> listProperties() {
+ return validNames;
+ }
+}
Deleted: trunk/src/org/griphyn/vdl/util/TriStateBoolean.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/TriStateBoolean.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/util/TriStateBoolean.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -1,66 +0,0 @@
-/*
- * Copyright 2012 University of Chicago
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/*
- * Created on Feb 7, 2007
- */
-package org.griphyn.vdl.util;
-
-public final class TriStateBoolean {
- public static final TriStateBoolean FALSE = new TriStateBoolean(0);
- public static final TriStateBoolean TRUE = new TriStateBoolean(1);
- public static final TriStateBoolean MAYBE = new TriStateBoolean(2);
-
- private int value;
-
- private TriStateBoolean(int value) {
- this.value = value;
- }
-
- public boolean equals(Object obj) {
- if (obj instanceof TriStateBoolean) {
- return value == ((TriStateBoolean) obj).value;
- }
- else {
- return false;
- }
- }
-
- public int hashCode() {
- return value;
- }
-
- public String toString() {
- switch(value) {
- case 0: return "false";
- case 1: return "true";
- default: return "maybe";
- }
- }
-
- public static TriStateBoolean valueOf(String value) {
- if ("false".equalsIgnoreCase(value) || "no".equalsIgnoreCase(value)) {
- return FALSE;
- }
- else if ("true".equalsIgnoreCase(value) || "yes".equalsIgnoreCase(value)) {
- return TRUE;
- }
- else {
- return MAYBE;
- }
- }
-}
Deleted: trunk/src/org/griphyn/vdl/util/VDL2Config.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/VDL2Config.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/util/VDL2Config.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -1,316 +0,0 @@
-/*
- * Copyright 2012 University of Chicago
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/*
- * Created on Dec 5, 2006
- */
-package org.griphyn.vdl.util;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-import org.apache.log4j.Logger;
-import org.globus.cog.karajan.util.BoundContact;
-import org.globus.common.CoGProperties;
-
-public class VDL2Config extends Properties {
-
- private static final long serialVersionUID = 1L;
-
- public static final Logger logger = Logger.getLogger(VDL2Config.class);
-
- public static final String CONFIG_FILE_NAME = "swift.properties";
- public static final String[] CONFIG_FILE_SEARCH_PATH = new String[] {
- System.getProperty("swift.home") + File.separator + "etc" + File.separator
- + CONFIG_FILE_NAME,
- System.getProperty("user.home") + File.separator + ".swift" + File.separator
- + CONFIG_FILE_NAME,
- System.getProperty("vds.home") + File.separator + "etc" + File.separator
- + CONFIG_FILE_NAME };
-
- private static VDL2Config config;
-
- public static VDL2Config getConfig() throws IOException {
- if(config == null) {
- config = getDefaultConfig();
- }
- return config.check();
- }
-
- public static synchronized VDL2Config getDefaultConfig() throws IOException {
- if (config == null) {
- config = new VDL2Config();
- for (int i = 0; i < CONFIG_FILE_SEARCH_PATH.length; i++) {
- config.load(CONFIG_FILE_SEARCH_PATH[i]);
- }
- }
- return config;
- }
-
- public static VDL2Config getConfig(String file) throws IOException {
- VDL2Config c;
- try {
- VDL2Config d = getConfig();
- c = new VDL2Config(d);
- }
- catch (Exception e) {
- c = new VDL2Config();
- }
- c.load(file);
- config = c;
- config.check();
- return config;
- }
-
- private List<String> files, tried;
- private Map<Object, ConfigPropertyType> types;
- private Map<Object, String> propertySource;
- private String currentFile;
-
- private VDL2Config() {
- files = new LinkedList<String>();
- tried = new LinkedList<String>();
- propertySource = new HashMap<Object, String>();
- types = new HashMap<Object, ConfigPropertyType>();
- put(VDL2ConfigProperties.POOL_FILE, "${swift.home}/etc/sites.xml", ConfigPropertyType.FILE);
- put(VDL2ConfigProperties.TC_FILE, "${swift.home}/etc/tc.data", ConfigPropertyType.FILE);
- put(VDL2ConfigProperties.LAZY_ERRORS, "false", ConfigPropertyType.BOOLEAN);
- put(VDL2ConfigProperties.CACHING_ALGORITHM, "LRU", ConfigPropertyType.STRING);
- put(VDL2ConfigProperties.CLUSTERING_ENABLED, "false", ConfigPropertyType.BOOLEAN);
- put(VDL2ConfigProperties.CLUSTERING_QUEUE_DELAY, "4", ConfigPropertyType.INT);
- put(VDL2ConfigProperties.CLUSTERING_MIN_TIME, "60", ConfigPropertyType.INT);
- put("throttle.submit", "4", ConfigPropertyType.INT);
- put("throttle.host.submit", "2", ConfigPropertyType.INT);
- put("throttle.transfers", "4", ConfigPropertyType.INT);
- put("throttle.file.operations", "8", ConfigPropertyType.INT);
- put("throttle.score.job.factor", "4", ConfigPropertyType.FLOAT);
- put(VDL2ConfigProperties.SITEDIR_KEEP, "false", ConfigPropertyType.BOOLEAN);
- put(VDL2ConfigProperties.PROVENANCE_LOG, "false", ConfigPropertyType.BOOLEAN);
-
- put("execution.retries", "0", ConfigPropertyType.INT);
-
- put("replication.enabled", "false", ConfigPropertyType.BOOLEAN);
- put("replication.min.queue.time", "60", ConfigPropertyType.INT);
- put("replication.limit", "3", ConfigPropertyType.INT);
- put("status.mode", "files", ConfigPropertyType.choices("files", "provider"));
- put("wrapper.parameter.mode", "args", ConfigPropertyType.choices("args", "files"));
- put("wrapper.invocation.mode", "absolute", ConfigPropertyType.choices("absolute", "relative"));
-
- // TODO what are the valid values here?
- put("cdm.broadcast.mode", "file");
- put("use.provider.staging", "false", ConfigPropertyType.BOOLEAN);
- put("use.wrapper.staging", "false", ConfigPropertyType.BOOLEAN);
- put("ticker.date.format", "", ConfigPropertyType.STRING);
- put("ticker.prefix", "Progress: ", ConfigPropertyType.STRING);
-
- put(VDL2ConfigProperties.FILE_GC_ENABLED, "true", ConfigPropertyType.BOOLEAN);
- put(VDL2ConfigProperties.DM_CHECKER, "on", ConfigPropertyType.ONOFF);
- }
-
- private VDL2Config(VDL2Config other) {
- this.putAll(other);
- this.files.addAll(other.files);
- this.propertySource.putAll(other.propertySource);
- }
-
- protected void load(String file) throws IOException {
- this.currentFile = file;
- tried.add(file);
- File f = new File(file);
- if (f.exists()) {
- files.add(file);
- super.load(new FileInputStream(f));
- }
- String hostname = getHostName();
- if (hostname != null) {
- CoGProperties.getDefault().setHostName(hostname);
- }
- String ip = getIP();
- if (ip != null) {
- CoGProperties.getDefault().setIPAddress(ip);
- }
- String tcpPortRange = getTCPPortRange();
- if (tcpPortRange != null) {
- CoGProperties.getDefault().put("tcp.port.range", tcpPortRange);
- }
- }
-
- public void validateProperties() {
- for (Map.Entry<Object, Object> e : this.entrySet()) {
- checkType(e.getKey(), e.getValue());
- }
- }
-
- protected VDL2Config check() throws IOException {
- if (files.size() == 0) {
- throw new FileNotFoundException("No Swift configuration file found. Tried " + tried);
- }
- else {
- return this;
- }
- }
-
- public synchronized Object put(Object key, Object value, ConfigPropertyType type) {
- types.put(key, type);
- return put(key, value);
- }
-
- /**
- * Overridden to do variable expansion. Variables will be expanded if there
- * is a system property with that name. Otherwise, the expansion will not
- * occur.
- */
- public synchronized Object put(Object key, Object value) {
- propertySource.put(key, currentFile);
- String svalue = (String) value;
- if (svalue.indexOf("${") == -1) {
- return super.put(key, value);
- }
- else {
- StringBuffer sb = new StringBuffer();
- int index = 0, last = 0;
- while (index >= 0) {
- index = svalue.indexOf("${", index);
- if (index >= 0) {
- if (last != index) {
- sb.append(svalue.substring(last, index));
- last = index;
- }
- int end = svalue.indexOf("}", index);
- if (end == -1) {
- sb.append(svalue.substring(index));
- break;
- }
- else {
- String name = svalue.substring(index + 2, end);
- String pval = System.getProperty(name);
- index = end + 1;
- if (pval == null) {
- continue;
- }
- else {
- sb.append(pval);
- last = index;
- }
- }
- }
- }
- sb.append(svalue.substring(last));
- value = sb.toString();
- return super.put(key, value);
- }
- }
-
- private void checkType(Object key, Object value) {
- ConfigPropertyType type = types.get(key);
- if (type != null) {
- type.checkValue((String) key, (String) value, propertySource.get(key));
- }
- }
-
- public String getPoolFile() {
- return getProperty(VDL2ConfigProperties.POOL_FILE);
- }
-
- public String getTCFile() {
- return getProperty(VDL2ConfigProperties.TC_FILE);
- }
-
- public String getIP() {
- return getProperty(VDL2ConfigProperties.IP_ADDRESS);
- }
-
- public String getHostName() {
- return getProperty(VDL2ConfigProperties.HOST_NAME);
- }
-
- public String getTCPPortRange() {
- return getProperty(VDL2ConfigProperties.TCP_PORT_RANGE);
- }
-
- public boolean getLazyErrors() {
- return Boolean.valueOf(getProperty(VDL2ConfigProperties.LAZY_ERRORS, "true")).booleanValue();
- }
-
- public TriStateBoolean getKickstartEnabled() {
- return TriStateBoolean.valueOf(getProperty(VDL2ConfigProperties.KICKSTART_ENABLED, "false"));
- }
-
- public boolean getSitedirKeep() {
- return Boolean.valueOf(getProperty(VDL2ConfigProperties.SITEDIR_KEEP, "true")).booleanValue();
- }
-
- private Boolean provenanceLogCached;
-
- public boolean getProvenanceLog() {
- if (provenanceLogCached == null) {
- provenanceLogCached = Boolean.valueOf(getProperty(VDL2ConfigProperties.PROVENANCE_LOG, "false"));
- }
- return provenanceLogCached;
- }
-
- public boolean isTracingEnabled() {
- return Boolean.valueOf(getProperty(VDL2ConfigProperties.TRACING_ENABLED, "false")).booleanValue();
- }
-
- public String getTickerDateFormat() {
- return getProperty("ticker.date.format");
- }
-
- public String getTickerPrefix() {
- return getProperty("ticker.prefix");
- }
-
- public String toString() {
- return "Swift configuration (" + files + "): " + super.toString();
- }
-
- public Object clone() {
- VDL2Config conf = new VDL2Config();
- conf.putAll(this);
- conf.files.addAll(files);
- conf.propertySource.putAll(propertySource);
- return conf;
- }
-
- public String getProperty(String name, BoundContact bc) {
- if(bc!=null) {
- if(logger.isDebugEnabled()) {
- logger.debug("Checking BoundContact "+bc+" for property "+name);
- }
- String prop = (String) bc.getProperty(name);
- if(prop != null) {
- return prop;
- }
- }
- if(logger.isDebugEnabled()) {
- logger.debug("Getting property "+name+" from global configuration");
- }
- return getProperty(name);
- }
-
- public void setCurrentFile(String f) {
- this.currentFile = f;
- }
-}
Deleted: trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2014-07-09 19:38:06 UTC (rev 7975)
+++ trunk/src/org/griphyn/vdl/util/VDL2ConfigProperties.java 2014-07-09 19:38:50 UTC (rev 7976)
@@ -1,171 +0,0 @@
-/*
- * Copyright 2012 University of Chicago
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/*
- * Created on Dec 23, 2006
- */
-package org.griphyn.vdl.util;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-public class VDL2ConfigProperties {
- public static final String POOL_FILE = "sites.file";
- public static final String TC_FILE = "tc.file";
- public static final String IP_ADDRESS = "ip.address";
- public static final String HOST_NAME = "hostname";
- public static final String TCP_PORT_RANGE = "tcp.port.range";
- public static final String LAZY_ERRORS = "lazy.errors";
- public static final String PGRAPH = "pgraph";
- public static final String PGRAPH_GRAPH_OPTIONS = "pgraph.graph.options";
- public static final String PGRAPH_NODE_OPTIONS = "pgraph.node.options";
- public static final String CACHING_ALGORITHM = "caching.algorithm";
- public static final String CLUSTERING_ENABLED = "clustering.enabled";
- public static final String CLUSTERING_QUEUE_DELAY = "clustering.queue.delay";
- public static final String CLUSTERING_MIN_TIME = "clustering.min.time";
- public static final String KICKSTART_ENABLED = "kickstart.enabled";
- public static final String KICKSTART_ALWAYS_TRANSFER = "kickstart.always.transfer";
- public static final String WRAPPERLOG_ALWAYS_TRANSFER = "wrapperlog.always.transfer";
- public static final String SITEDIR_KEEP = "sitedir.keep";
- public static final String PROVENANCE_LOG = "provenance.log";
- public static final String FILE_GC_ENABLED = "file.gc.enabled";
- public static final String TRACING_ENABLED = "tracing.enabled";
- public static final String DM_CHECKER = "mapping.checker";
- public static final Map<String, PropInfo> PROPERTIES;
-
- static {
- PROPERTIES = new TreeMap<String, PropInfo>();
- PROPERTIES.put(POOL_FILE, new PropInfo("file",
- "Points to the location of the sites.xml file"));
- PROPERTIES.put(TC_FILE, new PropInfo("file", "Points to the location of the tc.data file"));
- PROPERTIES.put(IP_ADDRESS, new PropInfo("aaa.bbb.ccc.ddd",
- "Can be used to specify a publicly reacheable IP address for "
- + "this machine which is generally used for Globus callbacks. "
- + "Normally this should be auto-detected, but if you have "
- + "multiple network cards or NAT then you may need to set this"));
- PROPERTIES.put(HOST_NAME, new PropInfo("string",
- "Can be used to specify a publicly reacheable DNS name or IP address for "
- + "this machine which is generally used for Globus callbacks. "
- + "Normally this should be auto-detected, but if you do "
- + "not have a public DNS name, you may want to set this."));
- PROPERTIES.put(TCP_PORT_RANGE, new PropInfo("start,end",
- "A TCP port range can be specified to "
- + "restrict the ports on which GRAM callback services are started. "
- + "This is likely needed if your submit host is behind a firewall, "
- + "in which case the firewall should be configured to allow "
- + "incoming connections on ports in the range."));
- PROPERTIES.put(LAZY_ERRORS, new PropInfo("true|false",
- "Use a lazy mode to deal with errors. When set to 'true' swift will proceed with the "
- + "execution until no more data can be derived because of "
- + "errors in dependent steps. If set to 'false', an error will "
- + "cause the execution to immediately stop"));
- PROPERTIES.put(PGRAPH, new PropInfo("true|false|<filename>",
- "Whether to generate a provenance "
- + "graph or not. If 'true' is used, the file name for the graph will "
- + "be chosen by swift."));
- PROPERTIES.put(PGRAPH_GRAPH_OPTIONS, new PropInfo("<string>",
- "Graph options to be passed to the .dot file. "
- + "These will appear in the 'graph' clause in the .dot file: "
- + "graph [<string>]; "));
- PROPERTIES.put(PGRAPH_NODE_OPTIONS, new PropInfo("<string>",
- "Node options to be passed to the .dot file."
- + "These will appear in the 'node' clause in the .dot file: "
- + "node [<string>]; "));
- PROPERTIES.put(CACHING_ALGORITHM, new PropInfo("[LRU]", "The algorithm to use for the "
- + "swift file caching mechanism. LRU is the only one available now."));
- PROPERTIES.put(CLUSTERING_ENABLED, new PropInfo("true|false",
- "Whether to enable clustering of small jobs. If enabled, jobs with a "
- + "max wall time which is less than the value of the "
- + CLUSTERING_MIN_TIME
- + " property will be clustered into one job which has a cummulative"
- + " max wall time greater or equal to the value of the "
- + CLUSTERING_MIN_TIME + " property."));
- PROPERTIES.put(CLUSTERING_QUEUE_DELAY, new PropInfo("<seconds>", "The delay at which "
- + "the clustering code scans the clustering queue. A job marked for clustering "
- + "will spend no more than the value of this property in the clustering queue."));
- PROPERTIES.put(CLUSTERING_MIN_TIME, new PropInfo("<seconds>", "The threshold determines "
- + " if a job as being clusterable. Also represents the minimum cummulative "
- + "wall time that a cluster will have."));
- PROPERTIES.put(KICKSTART_ENABLED, new PropInfo("<true|false|maybe>",
- "Controls the use of Kickstart by Swift. The \"maybe\" "
- + "value tells Swift to use Kickstart on sites where it is available."));
- PROPERTIES.put(
- KICKSTART_ALWAYS_TRANSFER,
- new PropInfo(
- "<true|false>",
- "If Kickstart is used (see \""
- + KICKSTART_ENABLED
- + "\"), it controls when "
- + "Kickstart records are transfered back to the submit host. If set to \"false\" "
- + "Swift will only transfer a Kickstart record for a job when the job fails. "
- + "If set to \"true\", Swift will transfer Kickstart records whether a job "
- + "fails or not."));
- PROPERTIES.put(
- WRAPPERLOG_ALWAYS_TRANSFER,
- new PropInfo(
- "<true|false>",
- "Controls when "
- + "wrapper logs are transfered back to the submit host. If set to \"false\" "
- + "Swift will only transfer a wrapper log for a job when the job fails. "
- + "If set to \"true\", Swift will transfer wrapper logs whether a job "
- + "fails or not."));
- PROPERTIES.put(
- SITEDIR_KEEP,
- new PropInfo(
- "<true|false>",
- "If set to true, keeps remote site run directory after execution has completed."));
-
-
- PROPERTIES.put(
- PROVENANCE_LOG,
- new PropInfo(
- "<true|false>",
- "If set to true, will record provenance information in the log file"));
-
- PROPERTIES.put(FILE_GC_ENABLED, new PropInfo("<true|false>", "Allows disabling the file garbage collector. " +
- "If set to false, files mapped by collectable mappers (such as the concurrent mapper) will not be " +
- "deleted when their swift variables go out of scope."));
-
- PROPERTIES.put(TRACING_ENABLED, new PropInfo("<true|false>", "Enables execution tracing. If set to 'true', " +
- "operations within swift such as iterations, invocations, assignments, and declarations, as well " +
- "as data dependencies will be logged. This comes at a cost in performance. It is therefore " +
- "disabled by default."));
-
- PROPERTIES.put(DM_CHECKER, new PropInfo("<on|off>", "Controls the run-time duplicate mapping checker " +
- "(which indetifies mapping conflicts). If set to 'on', the checker is enabled. When enabled, a record " +
- "of all mapped data is kept, so this comes at the expense of a slight memory leak. If set 'off', the" +
- " mapping checker is disabled. Enabled by default."));
- }
-
- public static Map<String, PropInfo> getPropertyDescriptions() {
- return PROPERTIES;
- }
-
- public static String getPropertyDescription(String name) {
- return PROPERTIES.get(name).desc;
- }
-
- public static class PropInfo {
- public final String validValues;
- public final String desc;
-
- public PropInfo(String validValues, String desc) {
- this.validValues = validValues;
- this.desc = desc;
- }
- }
-}
More information about the Swift-commit
mailing list