[Swift-commit] r8061 - in trunk/src/org/griphyn/vdl: karajan karajan/functions karajan/lib/swiftscript util
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Wed Jul 23 20:48:05 CDT 2014
Author: hategan
Date: 2014-07-23 20:48:05 -0500 (Wed, 23 Jul 2014)
New Revision: 8061
Modified:
trunk/src/org/griphyn/vdl/karajan/Loader.java
trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java
trunk/src/org/griphyn/vdl/util/ConfigTree.java
trunk/src/org/griphyn/vdl/util/SwiftConfig.java
trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java
Log:
implementation of latest config spec (see swift-devel emails 07/14/14-07/15/14)
Modified: trunk/src/org/griphyn/vdl/karajan/Loader.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-23 22:08:50 UTC (rev 8060)
+++ trunk/src/org/griphyn/vdl/karajan/Loader.java 2014-07-24 01:48:05 UTC (rev 8061)
@@ -78,7 +78,9 @@
public static final String ARG_HELP = "help";
public static final String ARG_VERSION = "version";
public static final String ARG_RESUME = "resume";
- public static final String ARG_INSTANCE_CONFIG = "config";
+ public static final String ARG_RUN_CONFIG = "config";
+ public static final String ARG_CONFIG_SEARCH_PATH = "configpath";
+ public static final String ARG_LIST_CONFIG = "listconfig";
public static final String ARG_SITELIST = "sitelist";
public static final String ARG_SITES_FILE = "sites.file";
public static final String ARG_TC_FILE = "tc.file";
@@ -319,6 +321,23 @@
"sites.xml and tc.data to a Swift configuration file.");
System.exit(1);
}
+ if (ap.isPresent(ARG_LIST_CONFIG)) {
+ SwiftConfig config = loadConfig(ap, getCommandLineProperties(ap));
+ String val = ap.getStringValue(ARG_LIST_CONFIG);
+ if ("files".equals(val)) {
+ System.out.println(config.toString(true, false));
+ System.exit(0);
+ }
+ else if ("full".equals(val)) {
+ System.out.println(config.toString(true, true));
+ System.exit(0);
+ }
+ else {
+ System.err.println("Invalid argument to -listconfig ('" + val + "'). " +
+ "Must be one of 'files' or 'full'");
+ System.exit(1);
+ }
+ }
if (!ap.hasValue(ArgumentParser.DEFAULT) && !ap.isPresent(ARG_EXECUTE)) {
System.out.println(loadVersion());
error("No Swift script specified");
@@ -560,15 +579,19 @@
private static SwiftConfig loadConfig(ArgumentParser ap, Map<String, Object> cmdLine) throws IOException {
SwiftConfig conf;
- if (ap.hasValue(ARG_INSTANCE_CONFIG)) {
- String configFile = ap.getStringValue(ARG_INSTANCE_CONFIG);
- conf = SwiftConfig.load(configFile, cmdLine);
- SwiftConfig.setDefault(conf);
+ conf = SwiftConfig.load(ap.getStringValue(ARG_RUN_CONFIG),
+ splitConfigSearchPath(ap.getStringValue(ARG_CONFIG_SEARCH_PATH)), cmdLine);
+ SwiftConfig.setDefault(conf);
+ return conf;
+ }
+
+ private static List<String> splitConfigSearchPath(String path) {
+ if (path == null) {
+ return null;
}
else {
- conf = (SwiftConfig) SwiftConfig.load().clone();
+ return Arrays.asList(path.split(File.pathSeparator));
}
- return conf;
}
private static Map<String, Object> getCommandLineProperties(ArgumentParser ap) {
@@ -608,12 +631,24 @@
ap.addOption(ARG_RESUME, "Resumes the execution using a log file",
"file", ArgumentParser.OPTIONAL);
- ap.addOption(ARG_INSTANCE_CONFIG,
- "Indicates the Swift configuration file to be used for this run." +
- " Properties in this configuration file will override the default properties. " +
- "If individual command line arguments are used for properties, they will override " +
- "the contents of this file.", "file",
+ ap.addOption(ARG_RUN_CONFIG,
+ "Indicates the run configuration file to be used for this run. If no custom " +
+ "configuration search path is specified, this will replace the default run " +
+ "configuration file ('./swift.conf') if it exists, or append to the configuration " +
+ "search path otherwise. If a search path is specified, the value of this argument " +
+ "will be appended to the search path.", "file",
ArgumentParser.OPTIONAL);
+ ap.addOption(ARG_CONFIG_SEARCH_PATH,
+ "Specifies a custom configuration search path. If supplied, the default configuration " +
+ "search path will be ignored and configurations will be loaded from files specified in " +
+ "the value of this argument. The files in the value must be separated using the " +
+ "operating system's path separator.", "file",
+ ArgumentParser.OPTIONAL);
+ ap.addOption(ARG_LIST_CONFIG,
+ "Lists details about the loaded configuration. 'files' lists only the configuration " +
+ "files used, while 'full' lists both the files and all the property values.",
+ "files|full",
+ ArgumentParser.OPTIONAL);
ap.addFlag(ARG_SITELIST, "Prints a list of sites available in the swift configuration");
ap.addFlag(ARG_VERBOSE,
"Increases the level of output that Swift produces on the console to include more detail " +
Modified: trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2014-07-23 22:08:50 UTC (rev 8060)
+++ trunk/src/org/griphyn/vdl/karajan/functions/ConfigProperty.java 2014-07-24 01:48:05 UTC (rev 8061)
@@ -137,8 +137,7 @@
prop = conf.getProperty(name);
}
if (prop == null) {
- throw new ExecutionException("Swift config property \"" + name + "\" not found in "
- + conf.getFileName());
+ throw new ExecutionException("Swift config property \"" + name + "\" not found");
}
else {
return prop;
Modified: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2014-07-23 22:08:50 UTC (rev 8060)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/Misc.java 2014-07-24 01:48:05 UTC (rev 8061)
@@ -301,6 +301,7 @@
}
}
+
public static class ExecSystem extends AbstractSingleValuedSwiftFunction {
private ArgRef<AbstractDataNode> input;
Modified: trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java 2014-07-23 22:08:50 UTC (rev 8060)
+++ trunk/src/org/griphyn/vdl/util/ConfigPropertyType.java 2014-07-24 01:48:05 UTC (rev 8061)
@@ -24,15 +24,15 @@
import com.typesafe.config.ConfigOrigin;
public abstract class ConfigPropertyType<T> {
- public static final ConfigPropertyType<Boolean> BOOLEAN = new CPTBoolean();
+ public static final ConfigPropertyType<Object> BOOLEAN = new CPTBoolean();
public static final ConfigPropertyType<String> STRING = new CPTString();
- public static final ConfigPropertyType<Integer> INT = new Int();
+ public static final ConfigPropertyType<Object> INT = new Int();
public static final ConfigPropertyType<Object> THROTTLE = new Throttle();
public static final ConfigPropertyType<String> PORT_RANGE = new PortRange();
- public static final ConfigPropertyType<Integer> STRICTLY_POSITIVE_INT = new SPInt();
- public static final ConfigPropertyType<Integer> POSITIVE_INT = new PInt();
- public static final ConfigPropertyType<Double> POSITIVE_FLOAT = new PFloat();
- public static final ConfigPropertyType<Double> FLOAT = new CPTFloat();
+ public static final ConfigPropertyType<Object> STRICTLY_POSITIVE_INT = new SPInt();
+ public static final ConfigPropertyType<Object> POSITIVE_INT = new PInt();
+ public static final ConfigPropertyType<Object> POSITIVE_FLOAT = new PFloat();
+ public static final ConfigPropertyType<Object> FLOAT = new CPTFloat();
public static final ConfigPropertyType<String> FILE = new CPTFile();
public static final ConfigPropertyType<String> TIME = new CPTTime();
public static final ConfigPropertyType<Object> OBJECT = new CPTObject();
@@ -51,6 +51,11 @@
public abstract Object checkValue(String propName, T value, ConfigOrigin loc);
public abstract ConfigPropertyType<?> getBaseType();
+
+ protected RuntimeException cannotConvert(ConfigOrigin loc, String propName, Object value, String toWhat) {
+ return new IllegalArgumentException(location(loc) + ":\n\tCannot convert value '" + value + "' for property '" +
+ propName + "' to " + toWhat);
+ }
private static String pp(Collection<String> c) {
StringBuilder sb = new StringBuilder();
@@ -126,10 +131,23 @@
}
}
- private static class Int extends ConfigPropertyType<Integer> {
+ private static class Int extends ConfigPropertyType<Object> {
@Override
- public Object checkValue(String propName, Integer value, ConfigOrigin loc) {
- return value;
+ public Object checkValue(String propName, Object value, ConfigOrigin loc) {
+ if (value instanceof String) {
+ try {
+ return Integer.parseInt((String) value);
+ }
+ catch (NumberFormatException e) {
+ throw cannotConvert(loc, propName, value, "integer");
+ }
+ }
+ else if (value instanceof Integer) {
+ return value;
+ }
+ else {
+ throw cannotConvert(loc, propName, value, "integer");
+ }
}
@Override
@@ -143,10 +161,18 @@
}
}
- private static class CPTBoolean extends ConfigPropertyType<Boolean> {
+ private static class CPTBoolean extends ConfigPropertyType<Object> {
@Override
- public Object checkValue(String propName, Boolean value, ConfigOrigin loc) {
- return value;
+ public Object checkValue(String propName, Object value, ConfigOrigin loc) {
+ if (value instanceof String) {
+ return Boolean.valueOf((String) value);
+ }
+ else if (value instanceof Boolean) {
+ return value;
+ }
+ else {
+ throw cannotConvert(loc, propName, value, "boolean");
+ }
}
@Override
@@ -160,20 +186,16 @@
}
}
- private static class SPInt extends ConfigPropertyType<Integer> {
+ private static class SPInt extends Int {
@Override
- public Object checkValue(String propName, Integer value, ConfigOrigin loc) {
- if (value <= 0) {
+ public Object checkValue(String propName, Object value, ConfigOrigin loc) {
+ Integer ivalue = (Integer) super.checkValue(propName, value, loc);
+ if (ivalue <= 0) {
throw new IllegalArgumentException(location(loc) + ":\n\tInvalid value '" + value + "' for property '" +
propName + "'. Must be a " + toString());
}
- return value;
+ return ivalue;
}
-
- @Override
- public ConfigPropertyType<?> getBaseType() {
- return INT;
- }
@Override
public String toString() {
@@ -181,20 +203,16 @@
}
}
- private static class PInt extends ConfigPropertyType<Integer> {
+ private static class PInt extends Int {
@Override
- public Object checkValue(String propName, Integer value, ConfigOrigin loc) {
- if (value < 0) {
+ public Object checkValue(String propName, Object value, ConfigOrigin loc) {
+ Integer ivalue = (Integer) super.checkValue(propName, value, loc);
+ if (ivalue < 0) {
throw new IllegalArgumentException(location(loc) + ":\n\tInvalid value '" + value + "' for property '" +
propName + "'. Must be a " + toString());
}
- return value;
+ return ivalue;
}
-
- @Override
- public ConfigPropertyType<?> getBaseType() {
- return INT;
- }
@Override
public String toString() {
@@ -209,6 +227,14 @@
if ("off".equals(value)) {
return Integer.MAX_VALUE;
}
+ else if (value instanceof String) {
+ try {
+ return Integer.parseInt((String) value);
+ }
+ catch (NumberFormatException e) {
+ throw cannotConvert(loc, propName, value, "integer");
+ }
+ }
else if (value instanceof Integer) {
Integer i = (Integer) value;
if (i > 0) {
@@ -230,20 +256,16 @@
}
}
- private static class PFloat extends ConfigPropertyType<Double> {
+ private static class PFloat extends CPTFloat {
@Override
- public Object checkValue(String propName, Double value, ConfigOrigin loc) {
- if (value < 0) {
+ public Object checkValue(String propName, Object value, ConfigOrigin loc) {
+ Double dvalue = (Double) super.checkValue(propName, value, loc);
+ if (dvalue < 0) {
throw new IllegalArgumentException(location(loc) + ":\n\tInvalid value '" + value + "' for property '" +
propName + "'. Must be a " + toString());
}
- return value;
+ return dvalue;
}
-
- @Override
- public ConfigPropertyType<?> getBaseType() {
- return FLOAT;
- }
@Override
public String toString() {
@@ -251,10 +273,23 @@
}
}
- private static class CPTFloat extends ConfigPropertyType<Double> {
+ private static class CPTFloat extends ConfigPropertyType<Object> {
@Override
- public Object checkValue(String propName, Double value, ConfigOrigin loc) {
- return value;
+ public Object checkValue(String propName, Object value, ConfigOrigin loc) {
+ if (value instanceof String) {
+ try {
+ return Double.parseDouble((String) value);
+ }
+ catch (NumberFormatException e) {
+ throw cannotConvert(loc, propName, value, "number");
+ }
+ }
+ else if (value instanceof Double) {
+ return value;
+ }
+ else {
+ throw cannotConvert(loc, propName, value, "number");
+ }
}
@Override
@@ -263,7 +298,7 @@
}
}
- public static class Interval extends ConfigPropertyType<Double> {
+ public static class Interval extends CPTFloat {
private double l, h;
public Interval(double l, double h) {
@@ -272,18 +307,14 @@
}
@Override
- public Object checkValue(String propName, Double value, ConfigOrigin loc) {
- if (value < l || value > h) {
+ public Object checkValue(String propName, Object value, ConfigOrigin loc) {
+ Double dvalue = (Double) super.checkValue(propName, value, loc);
+ if (dvalue < l || dvalue > h) {
throw new IllegalArgumentException(location(loc) + ":\n\tInvalid value '" + value + "' for property '" +
propName + "'. Must be a " + toString());
}
- return value;
+ return dvalue;
}
-
- @Override
- public ConfigPropertyType<?> getBaseType() {
- return FLOAT;
- }
@Override
public String toString() {
Modified: trunk/src/org/griphyn/vdl/util/ConfigTree.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/ConfigTree.java 2014-07-23 22:08:50 UTC (rev 8060)
+++ trunk/src/org/griphyn/vdl/util/ConfigTree.java 2014-07-24 01:48:05 UTC (rev 8061)
@@ -10,12 +10,14 @@
package org.griphyn.vdl.util;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
+import java.util.TreeSet;
public class ConfigTree<T> {
@@ -49,7 +51,7 @@
}
}
- public Object get() {
+ public T get() {
return value;
}
@@ -202,32 +204,18 @@
}
}
- private void toString(StringBuilder sb, int level, String k) {
+ private void toString(StringBuilder sb, int level, String k, String full, boolean sort, ValueFormatter f) {
if (nodes == null || nodes.isEmpty()) {
- for (int i = 0; i < level; i++) {
- sb.append('\t');
- }
- if (value != null) {
- sb.append(k);
- sb.append(": ");
- if (value instanceof String) {
- sb.append('\"');
- sb.append(value);
- sb.append('\"');
- }
- else {
- sb.append(value);
- }
- sb.append('\n');
- }
+ f.format(k, full, value, level, sb);
}
else if (nodes.size() == 1) {
String key = nodes.keySet().iterator().next();
if (k == null) {
- nodes.values().iterator().next().toString(sb, level, key);
+ nodes.values().iterator().next().toString(sb, level, key, cat(full, key), sort, f);
}
else {
- nodes.values().iterator().next().toString(sb, level, k + "." + key);
+ String nkey = cat(k, key);
+ nodes.values().iterator().next().toString(sb, level, nkey, cat(full, key), sort, f);
}
}
else {
@@ -239,9 +227,16 @@
sb.append(' ');
}
sb.append("{\n");
- for (Map.Entry<String, Node<T>> e : nodes.entrySet()) {
- e.getValue().toString(sb, level + 1, e.getKey());
+ Collection<String> keys;
+ if (sort) {
+ keys = new TreeSet<String>(nodes.keySet());
}
+ else {
+ keys = nodes.keySet();
+ }
+ for (String key : keys) {
+ nodes.get(key).toString(sb, level + 1, key, cat(full, key), sort, f);
+ }
for (int i = 0; i < level; i++) {
sb.append('\t');
}
@@ -249,6 +244,15 @@
}
}
+ private String cat(String full, String key) {
+ if (full == null) {
+ return key;
+ }
+ else {
+ return full + "." + key;
+ }
+ }
+
public List<String> getLeafPaths() {
List<String> l = new ArrayList<String>();
getLeafPaths(l, null);
@@ -257,7 +261,7 @@
public String toString() {
StringBuilder sb = new StringBuilder();
- toString(sb, 0, null);
+ toString(sb, 0, null, null, false, DEFAULT_VALUE_FORMATTER);
return sb.toString();
}
}
@@ -315,8 +319,40 @@
}
public String toString() {
+ return toString(false, DEFAULT_VALUE_FORMATTER);
+ }
+
+ public String toString(boolean sort, ValueFormatter f) {
StringBuilder sb = new StringBuilder();
- root.toString(sb, 0, null);
+ root.toString(sb, 0, null, null, sort, f);
return sb.toString();
}
+
+ public interface ValueFormatter {
+ void format(String key, String full, Object value, int indentationLevel, StringBuilder sb);
+ }
+
+ public static class DefaultValueFormatter implements ValueFormatter {
+ @Override
+ public void format(String key, String full, Object value, int indentationLevel, StringBuilder sb) {
+ for (int i = 0; i < indentationLevel; i++) {
+ sb.append('\t');
+ }
+ if (value != null) {
+ sb.append(key);
+ sb.append(": ");
+ if (value instanceof String) {
+ sb.append('\"');
+ sb.append(value);
+ sb.append('\"');
+ }
+ else {
+ sb.append(value);
+ }
+ sb.append('\n');
+ }
+ }
+ }
+
+ public static final ValueFormatter DEFAULT_VALUE_FORMATTER = new DefaultValueFormatter();
}
Modified: trunk/src/org/griphyn/vdl/util/SwiftConfig.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/SwiftConfig.java 2014-07-23 22:08:50 UTC (rev 8060)
+++ trunk/src/org/griphyn/vdl/util/SwiftConfig.java 2014-07-24 01:48:05 UTC (rev 8061)
@@ -16,7 +16,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedSet;
import java.util.TreeSet;
import org.apache.log4j.Logger;
@@ -40,6 +39,7 @@
import com.typesafe.config.ConfigIncludeContext;
import com.typesafe.config.ConfigIncluder;
import com.typesafe.config.ConfigObject;
+import com.typesafe.config.ConfigOrigin;
import com.typesafe.config.ConfigParseOptions;
import com.typesafe.config.ConfigSyntax;
import com.typesafe.config.ConfigValue;
@@ -51,7 +51,9 @@
public static final boolean CHECK_DYNAMIC_NAMES = true;
public static final boolean BUILD_CHECK = true;
- public static final List<String> DEFAULT_LOCATIONS;
+ public static String DIST_CONF;
+ public static String SITE_CONF;
+ public static String USER_CONF;
public enum Key {
DM_CHECKER("mappingCheckerEnabled"),
@@ -81,10 +83,7 @@
static {
SCHEMA = new SwiftConfigSchema();
- DEFAULT_LOCATIONS = new ArrayList<String>();
- if (System.getenv("SWIFT_SITE_CONF") != null) {
- DEFAULT_LOCATIONS.add(System.getenv("SWIFT_SITE_CONF"));
- }
+
String swiftHome = System.getProperty("swift.home");
if (swiftHome == null) {
swiftHome = System.getProperty("swift.home");
@@ -93,11 +92,18 @@
}
}
- DEFAULT_LOCATIONS.add(swiftHome + File.separator +
- "etc" + File.separator + "swift.conf");
+ DIST_CONF = makePath(swiftHome, "etc", "swift.conf");
+ if (System.getenv("SWIFT_SITE_CONF") != null) {
+ SITE_CONF = System.getenv("SWIFT_SITE_CONF");
+ }
+
+ File userConf = new File(makePath(System.getProperty("user.home"), ".swift", "swift.conf"));
+ if (userConf.exists()) {
+ USER_CONF = userConf.getAbsolutePath();
+ }
+
// check keys against schema
-
for (Key k : Key.values()) {
if (!SCHEMA.isNameValid(k.propName)) {
throw new IllegalArgumentException("Invalid property name for config key '" + k + "': " + k.propName);
@@ -115,11 +121,23 @@
}
}
+ public static class ValueLocationPair {
+ public final Object value;
+ public final ConfigOrigin loc;
+
+ public ValueLocationPair(Object value, ConfigOrigin loc) {
+ this.value = value;
+ this.loc = loc;
+ }
+ }
+
private static class IncluderWrapper implements ConfigIncluder {
private final ConfigIncluder d;
+ private final List<String> loadedFiles;
- public IncluderWrapper(ConfigIncluder d) {
+ public IncluderWrapper(ConfigIncluder d, List<String> loadedFiles) {
this.d = d;
+ this.loadedFiles = loadedFiles;
}
@Override
@@ -136,6 +154,7 @@
what = what.substring(0, b) + resolve(var) + what.substring(e + 1);
b = what.indexOf("${");
}
+ loadedFiles.add(new File(what).getAbsolutePath());
return ConfigFactory.parseFile(new File(what)).root();
}
@@ -153,28 +172,119 @@
return v;
}
}
-
- public static SwiftConfig load(String fileName) {
- return load(fileName, null);
+
+ private static String makePath(String... els) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < els.length; i++) {
+ if (i != 0 && (sb.charAt(sb.length() - 1) != File.separatorChar)) {
+ sb.append(File.separator);
+ }
+ sb.append(els[i]);
+ }
+ return sb.toString();
}
- public static SwiftConfig load(String fileName, Map<String, Object> override) {
+ public static SwiftConfig load(String cmdLineConfig, List<String> configSearchPath, Map<String, Object> cmdLineOptions) {
+ List<String> loadedFiles = new ArrayList<String>();
+ List<String> loadedFileIndices = new ArrayList<String>();
+
ConfigParseOptions opt = ConfigParseOptions.defaults();
- opt = opt.setIncluder(new IncluderWrapper(opt.getIncluder())).
+ opt = opt.setIncluder(new IncluderWrapper(opt.getIncluder(), loadedFiles)).
setSyntax(ConfigSyntax.CONF).setAllowMissing(false);
- Config conf = ConfigFactory.parseFile(new File(fileName), opt);
- if (override != null) {
- Config oconf = ConfigFactory.parseMap(override, "<command line>");
+
+ Config conf;
+
+ if (configSearchPath == null) {
+ conf = loadNormal(cmdLineConfig, opt, loadedFiles, loadedFileIndices);
+ }
+ else {
+ conf = loadFromSearchPath(configSearchPath, cmdLineConfig, opt,
+ loadedFiles, loadedFileIndices);
+ }
+
+ if (cmdLineOptions != null) {
+ Config oconf = ConfigFactory.parseMap(cmdLineOptions, "<Command Line>");
conf = oconf.withFallback(conf);
+ loadedFiles.add("<Command Line>");
+ loadedFileIndices.add("C");
}
+
conf = conf.resolveWith(getSubstitutions());
- ConfigTree<Object> out = SCHEMA.validate(conf);
- SwiftConfig sc = new SwiftConfig();
- sc.setFileName(fileName);
+ ConfigTree<ValueLocationPair> out = SCHEMA.validate(conf);
+ SwiftConfig sc = new SwiftConfig(loadedFiles, loadedFileIndices);
sc.build(out);
return sc;
}
+ private static Config loadFromSearchPath(List<String> configSearchPath, String cmdLineConfig,
+ ConfigParseOptions opt, List<String> loadedFiles, List<String> loadedFileIndices) {
+ Config conf = null;
+
+ int index = 1;
+ for (String c : configSearchPath) {
+ conf = loadOne(c, conf, opt);
+ loadedFiles.add(c);
+ loadedFileIndices.add(String.valueOf(index++));
+ }
+
+ if (cmdLineConfig != null) {
+ conf = loadOne(cmdLineConfig, conf, opt);
+ loadedFiles.add(cmdLineConfig);
+ loadedFileIndices.add("R");
+ }
+
+ return conf;
+ }
+
+ private static Config loadNormal(String cmdLineConfig, ConfigParseOptions opt,
+ List<String> loadedFiles, List<String> loadedFileIndices) {
+ Config conf;
+
+ conf = loadOne(DIST_CONF, null, opt);
+ loadedFiles.add(DIST_CONF);
+ loadedFileIndices.add("D");
+
+ if (SITE_CONF != null) {
+ conf = loadOne(SITE_CONF, conf, opt);
+ loadedFiles.add(SITE_CONF);
+ loadedFileIndices.add("S");
+ }
+
+ if (USER_CONF != null) {
+ conf = loadOne(USER_CONF, conf, opt);
+ loadedFiles.add(USER_CONF);
+ loadedFileIndices.add("U");
+ }
+
+ if (cmdLineConfig == null) {
+ File runConf = new File("swift.conf");
+ if (runConf.exists()) {
+ conf = loadOne(runConf.getPath(), conf, opt);
+ loadedFiles.add(runConf.getPath());
+ loadedFileIndices.add("R");
+ }
+ }
+
+ if (cmdLineConfig != null) {
+ conf = loadOne(cmdLineConfig, conf, opt);
+ loadedFiles.add(cmdLineConfig);
+ loadedFileIndices.add("R");
+ }
+
+ return conf;
+ }
+
+ private static Config loadOne(String fileName, Config conf, ConfigParseOptions opt) {
+ File f = new File(fileName);
+ Config nconf = ConfigFactory.parseFile(f, opt);
+ if (conf == null) {
+ return nconf;
+ }
+ else {
+ return nconf.withFallback(conf);
+ }
+ }
+
private static Config getSubstitutions() {
Map<String, Object> m = new HashMap<String, Object>();
@@ -184,21 +294,12 @@
return ConfigFactory.parseMap(m).withFallback(ConfigFactory.parseProperties(System.getProperties()));
}
-
- public static SwiftConfig load() {
- for (String loc : DEFAULT_LOCATIONS) {
- if (new File(loc).exists()) {
- return load(loc);
- }
- }
- throw new IllegalStateException("Could not find swift configuration file");
- }
private static SwiftConfig _default;
public synchronized static SwiftConfig getDefault() {
if (_default == null) {
- _default = load();
+ throw new IllegalStateException("No default Swift configuration set");
}
return _default;
}
@@ -209,14 +310,18 @@
private SwiftContactSet definedSites;
private SwiftContactSet sites;
- private ConfigTree<Object> tree;
+ private ConfigTree<ValueLocationPair> tree;
private Map<String, Object> flat;
private String fileName;
+ private List<String> usedFiles;
+ private List<String> usedFileIndices;
- public SwiftConfig() {
+ public SwiftConfig(List<String> usedFiles, List<String> usedFileIndices) {
definedSites = new SwiftContactSet();
sites = new SwiftContactSet();
flat = new HashMap<String, Object>();
+ this.usedFiles = usedFiles;
+ this.usedFileIndices = usedFileIndices;
}
public SwiftContactSet getSites() {
@@ -239,24 +344,16 @@
flat.put(key, value);
}
- public String getFileName() {
- return fileName;
- }
-
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
-
@SuppressWarnings("unchecked")
- private void build(ConfigTree<Object> tree) {
+ private void build(ConfigTree<ValueLocationPair> tree) {
this.tree = tree;
List<String> sites = null;
if (BUILD_CHECK) {
checkKey("sites");
}
- for (Map.Entry<String, ConfigTree.Node<Object>> e : tree.entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> e : tree.entrySet()) {
if (e.getKey().equals("site")) {
- for (Map.Entry<String, ConfigTree.Node<Object>> f : e.getValue().entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> f : e.getValue().entrySet()) {
site(definedSites, f.getKey(), f.getValue());
}
}
@@ -280,7 +377,7 @@
this.sites.getApplications().putAll(definedSites.getApplications());
for (String leaf : tree.getLeafPaths()) {
- flat.put(leaf, tree.get(leaf));
+ flat.put(leaf, tree.get(leaf).value);
}
}
@@ -295,16 +392,16 @@
}
}
- private void apps(SwiftContact sc, ConfigTree.Node<Object> n) {
+ private void apps(SwiftContact sc, ConfigTree.Node<ValueLocationPair> n) {
/*
* app."*" {
* ...
* }
*/
- for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> e : n.entrySet()) {
String k = e.getKey();
- ConfigTree.Node<Object> c = e.getValue();
+ ConfigTree.Node<ValueLocationPair> c = e.getValue();
if (e.getKey().equals("ALL")) {
sc.addApplication(app("*", e.getValue()));
@@ -351,7 +448,7 @@
}
}
- private Application app(String name, ConfigTree.Node<Object> n) {
+ private Application app(String name, ConfigTree.Node<ValueLocationPair> n) {
/*
* app."*" {
* executable: "?String"
@@ -374,9 +471,9 @@
checkKey("app.*.options.jobProject");
checkKey("app.*.options.jobQueue");
}
- for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> e : n.entrySet()) {
String k = e.getKey();
- ConfigTree.Node<Object> c = e.getValue();
+ ConfigTree.Node<ValueLocationPair> c = e.getValue();
if (k.equals("executable")) {
app.setExecutable(getString(c));
@@ -410,15 +507,15 @@
}
- private List<KVPair> envs(Node<Object> n) {
+ private List<KVPair> envs(Node<ValueLocationPair> n) {
List<KVPair> l = new ArrayList<KVPair>();
- for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> e : n.entrySet()) {
l.add(new KVPair(e.getKey(), getString(e.getValue())));
}
return l;
}
- private void site(SwiftContactSet sites, String name, ConfigTree.Node<Object> n) {
+ private void site(SwiftContactSet sites, String name, ConfigTree.Node<ValueLocationPair> n) {
try {
SwiftContact sc = new SwiftContact(name);
@@ -436,9 +533,9 @@
sc.setProperty("sysinfo", getString(n, "OS"));
}
- for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> e : n.entrySet()) {
String ctype = e.getKey();
- ConfigTree.Node<Object> c = e.getValue();
+ ConfigTree.Node<ValueLocationPair> c = e.getValue();
if (ctype.equals("execution")) {
sc.addService(execution(c));
@@ -469,7 +566,7 @@
}
}
- private void staging(SwiftContact sc, Node<Object> n) {
+ private void staging(SwiftContact sc, Node<ValueLocationPair> n) {
String staging = getString(n);
if (BUILD_CHECK) {
checkValue("site.*.staging",
@@ -516,14 +613,14 @@
}
}
- private Service filesystem(Node<Object> c) throws InvalidProviderException, ProviderMethodException {
+ private Service filesystem(Node<ValueLocationPair> c) throws InvalidProviderException, ProviderMethodException {
Service s = new ServiceImpl();
s.setType(Service.FILE_OPERATION);
fileService(c, s);
return s;
}
- private void fileService(Node<Object> n, Service s) throws InvalidProviderException, ProviderMethodException {
+ private void fileService(Node<ValueLocationPair> n, Service s) throws InvalidProviderException, ProviderMethodException {
String provider = null;
String url = null;
if (BUILD_CHECK) {
@@ -531,9 +628,9 @@
checkKey("site.*.filesystem.URL");
checkKey("site.*.filesystem.options");
}
- for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> e : n.entrySet()) {
String k = e.getKey();
- ConfigTree.Node<Object> c = e.getValue();
+ ConfigTree.Node<ValueLocationPair> c = e.getValue();
if (k.equals("type")) {
provider = getString(c);
@@ -542,7 +639,7 @@
url = getString(c);
}
else if (k.equals("options")) {
- for (Map.Entry<String, ConfigTree.Node<Object>> f : e.getValue().entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> f : e.getValue().entrySet()) {
s.setAttribute(f.getKey(), getObject(f.getValue()));
}
}
@@ -557,13 +654,13 @@
}
- private Service execution(ConfigTree.Node<Object> n) throws InvalidProviderException, ProviderMethodException {
+ private Service execution(ConfigTree.Node<ValueLocationPair> n) throws InvalidProviderException, ProviderMethodException {
ExecutionService s = new ExecutionServiceImpl();
execService(n, s);
return s;
}
- private void execService(Node<Object> n, Service s) throws InvalidProviderException, ProviderMethodException {
+ private void execService(Node<ValueLocationPair> n, Service s) throws InvalidProviderException, ProviderMethodException {
String provider = null;
String url = null;
if (BUILD_CHECK) {
@@ -572,9 +669,9 @@
checkKey("site.*.execution.jobManager");
checkKey("site.*.execution.options");
}
- for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> e : n.entrySet()) {
String k = e.getKey();
- ConfigTree.Node<Object> c = e.getValue();
+ ConfigTree.Node<ValueLocationPair> c = e.getValue();
if (k.equals("type")) {
provider = getString(c);
@@ -599,7 +696,7 @@
}
}
- private void execOptions(ExecutionService s, Node<Object> n) {
+ private void execOptions(ExecutionService s, Node<ValueLocationPair> n) {
if (BUILD_CHECK) {
checkKey("site.*.execution.options.jobProject");
checkKey("site.*.execution.options.maxJobs");
@@ -609,9 +706,9 @@
checkKey("site.*.execution.options.jobOptions");
}
- for (Map.Entry<String, ConfigTree.Node<Object>> e : n.entrySet()) {
+ for (Map.Entry<String, ConfigTree.Node<ValueLocationPair>> e : n.entrySet()) {
String k = e.getKey();
- ConfigTree.Node<Object> c = e.getValue();
+ ConfigTree.Node<ValueLocationPair> c = e.getValue();
if (k.equals("jobProject")) {
@@ -643,16 +740,16 @@
}
}
- private String getString(Node<Object> c) {
- return (String) c.get();
+ private String getString(Node<ValueLocationPair> c) {
+ return (String) c.get().value;
}
- private Object getObject(ConfigTree.Node<Object> c) {
- return c.get();
+ private Object getObject(ConfigTree.Node<ValueLocationPair> c) {
+ return c.get().value;
}
- private String getString(ConfigTree.Node<Object> c, String key) {
- return (String) c.get(key);
+ private String getString(ConfigTree.Node<ValueLocationPair> c, String key) {
+ return (String) c.get(key).value;
}
private Object getObject(ConfigTree.Node<Object> c, String key) {
@@ -671,23 +768,82 @@
}
public String toString() {
+ return toString(true, true);
+ }
+
+ public String toString(boolean files, boolean values) {
StringBuilder sb = new StringBuilder();
- SortedSet<String> s = new TreeSet<String>(flat.keySet());
- for (String k : s) {
- sb.append(k);
+ if (files) {
+ for (int i = 0; i < usedFiles.size(); i++) {
+ sb.append("[" + usedFileIndices.get(i) + "] " + usedFiles.get(i) + "\n");
+ }
+ }
+ if (values) {
+ sb.append(tree.toString(true, new LocationFormatter(usedFiles, usedFileIndices, tree)));
+ }
+ return sb.toString();
+ }
+
+ private static class LocationFormatter extends ConfigTree.DefaultValueFormatter {
+ private List<String> files, fileIndices;
+ private ConfigTree<ValueLocationPair> tree;
+
+ public LocationFormatter(List<String> files, List<String> fileIndices, ConfigTree<ValueLocationPair> tree) {
+ this.files = files;
+ this.fileIndices = fileIndices;
+ this.tree = tree;
+ }
+
+ @Override
+ public void format(String key, String full, Object value, int indentationLevel, StringBuilder sb) {
+ ValueLocationPair p = tree.get(full);
+ if (p.loc == null) {
+ // properties with default values from the schema, so don't print those
+ return;
+ }
+ int start = sb.length();
+ for (int i = 0; i < indentationLevel; i++) {
+ sb.append(" ");
+ }
+ sb.append(key);
sb.append(": ");
- Object o = flat.get(k);
- if (o instanceof String) {
- sb.append('"');
- sb.append(o);
- sb.append('"');
+
+ if (p.value instanceof String) {
+ sb.append("\"");
}
+ sb.append(p.value);
+ if (p.value instanceof String) {
+ sb.append("\"");
+ }
+
+ int width = sb.length() - start;
+
+ for (int i = 0; i < 60 - width; i++) {
+ sb.append(' ');
+ }
+
+ sb.append("# [");
+ int index;
+ if (p.loc.filename() == null) {
+ // command line
+ index = files.size() - 1;
+ }
else {
- sb.append(o);
+ index = files.indexOf(p.loc.filename());
}
+ if (index == -1) {
+ sb.append('?');
+ }
+ else {
+ sb.append(fileIndices.get(index));
+ }
+ sb.append("]");
+ if (p.loc.filename() != null) {
+ sb.append(" line ");
+ sb.append(p.loc.lineNumber());
+ }
sb.append('\n');
}
- return sb.toString();
}
private void check(String name) {
Modified: trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java
===================================================================
--- trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java 2014-07-23 22:08:50 UTC (rev 8060)
+++ trunk/src/org/griphyn/vdl/util/SwiftConfigSchema.java 2014-07-24 01:48:05 UTC (rev 8061)
@@ -16,6 +16,8 @@
import java.util.Map;
import java.util.Set;
+import org.griphyn.vdl.util.SwiftConfig.ValueLocationPair;
+
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigOrigin;
@@ -173,8 +175,8 @@
return o.filename() + ":" + o.lineNumber();
}
- public ConfigTree<Object> validate(Config conf) {
- ConfigTree<Object> validated = new ConfigTree<Object>();
+ public ConfigTree<ValueLocationPair> validate(Config conf) {
+ ConfigTree<ValueLocationPair> validated = new ConfigTree<ValueLocationPair>();
// build a tree of the actual config so we can easily check them for missing properties
ConfigTree<Boolean> confTree = new ConfigTree<Boolean>();
@@ -194,7 +196,7 @@
throw new IllegalStateException("Missing type for key " + k);
}
Object value = checkValue(k, e.getValue(), i.type);
- validated.put(k, value);
+ validated.put(k, new ValueLocationPair(value, e.getValue().origin()));
}
// check for missing things
@@ -210,14 +212,14 @@
return validated;
}
- private String findMissing(String key, ConfigTree<Boolean> confTree, Info i, ConfigTree<Object> validated) {
+ private String findMissing(String key, ConfigTree<Boolean> confTree, Info i, ConfigTree<ValueLocationPair> validated) {
List<String> found = confTree.expandWildcards(key, STAR);
for (String f : found) {
if (!confTree.hasKey(f)) {
if (i.optional) {
if (i.value != null) {
- validated.put(f, i.value);
+ validated.put(f, new ValueLocationPair(i.value, null));
}
}
else if (!parentsAreOptional(key, confTree)) {
@@ -237,8 +239,18 @@
return false;
}
Info i = info.get(k);
- if (i != null && !i.optional && !confTree.hasKey(k)) {
- return false;
+ if (i != null) {
+ if (i.optional) {
+ if (confTree.hasKey(k)) {
+ // continue checking parents
+ }
+ else {
+ return true;
+ }
+ }
+ else {
+ return false;
+ }
}
}
return true;
@@ -270,9 +282,7 @@
Object v = value.unwrapped();
switch (value.valueType()) {
case STRING:
- if (t.getBaseType() != ConfigPropertyType.STRING && t.getBaseType() != ConfigPropertyType.OBJECT) {
- throw invalidValue(value, k, v, t.getBaseType());
- }
+ // allow auto-conversion from string
return t.check(k, value.unwrapped(), value.origin());
case NUMBER:
if (t.getBaseType() != ConfigPropertyType.INT && t.getBaseType() != ConfigPropertyType.FLOAT) {
More information about the Swift-commit
mailing list