[Swift-commit] r6390 - in branches/faster/src/org/griphyn/vdl: engine mapping mapping/file
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Sun Mar 17 23:49:05 CDT 2013
Author: hategan
Date: 2013-03-17 23:49:05 -0500 (Sun, 17 Mar 2013)
New Revision: 6390
Added:
branches/faster/src/org/griphyn/vdl/mapping/file/DirectoryScanner.java
Modified:
branches/faster/src/org/griphyn/vdl/engine/Karajan.java
branches/faster/src/org/griphyn/vdl/mapping/AbsFile.java
branches/faster/src/org/griphyn/vdl/mapping/AbstractMapper.java
branches/faster/src/org/griphyn/vdl/mapping/Mapper.java
branches/faster/src/org/griphyn/vdl/mapping/MapperFactory.java
branches/faster/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/CSVMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/ExternalMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/SimpleFileMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java
branches/faster/src/org/griphyn/vdl/mapping/file/TestMapper.java
Log:
merged mapper parameter checking from 0.94 and added deep globbing (including /**/ for arbitrary depth subdirs)
Modified: branches/faster/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/engine/Karajan.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/engine/Karajan.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -65,6 +65,7 @@
import org.griphyn.vdl.engine.VariableScope.WriteType;
import org.griphyn.vdl.karajan.CompilationException;
import org.griphyn.vdl.karajan.Loader;
+import org.griphyn.vdl.mapping.MapperFactory;
import org.griphyn.vdl.toolkit.VDLt2VDLx;
import org.griphyn.vdl.type.NoSuchTypeException;
import org.safehaus.uuid.UUIDGenerator;
@@ -473,7 +474,9 @@
if (mapping != null) {
StringTemplate mappingST = new StringTemplate("mapping");
- mappingST.setAttribute("descriptor", mapping.getDescriptor());
+ String mapperType = mapping.getDescriptor();
+ mappingST.setAttribute("descriptor", mapperType);
+ checkMapperParams(mapperType, mapping);
for (int i = 0; i < mapping.sizeOfParamArray(); i++) {
Param param = mapping.getParamArray(i);
mappingST.setAttribute("params", mappingParameter(param, scope));
@@ -499,6 +502,27 @@
scope.bodyTemplate.setAttribute("declarations", variableST);
}
+ private void checkMapperParams(String mapperType, Mapping mapping) throws CompilationException {
+ if (!MapperFactory.isValidMapperType(mapperType)) {
+ throw new CompilationException("Unknown mapper type: '" + mapperType + "'");
+ }
+
+ Set<String> validParams = MapperFactory.getValidParams(mapperType);
+ if (validParams == null && mapping.sizeOfParamArray() > 0) {
+ throw new CompilationException(mapperType + " does not support any parameters");
+ }
+ if (validParams.contains("*")) {
+ // mapper accepts any parameter (e.g. external_mapper)
+ return;
+ }
+ for (int i = 0; i < mapping.sizeOfParamArray(); i++) {
+ Param param = mapping.getParamArray(i);
+ if (!validParams.contains(param.getName())) {
+ throw new CompilationException(mapperType + " does not support a '" + param.getName() + "' parameter");
+ }
+ }
+ }
+
private StringTemplate mappingParameter(Param param, VariableScope scope) throws CompilationException {
StringTemplate paramST = template("swift_parameter");
paramST.setAttribute("name", param.getName());
Modified: branches/faster/src/org/griphyn/vdl/mapping/AbsFile.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/AbsFile.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/AbsFile.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -44,7 +44,7 @@
if (pi == -1) {
protocol = "file";
host = "localhost";
- path = url;
+ path = normalize(url);
}
else {
protocol = url.substring(0, pi);
@@ -54,7 +54,7 @@
if (rp.startsWith("localhost/")) {
rp = rp.substring("localhost/".length());
}
- path = rp;
+ path = normalize(rp);
}
else {
int si = url.indexOf('/', pi + 3);
@@ -64,13 +64,19 @@
}
else {
host = url.substring(pi + 3, si);
- path = url.substring(si + 1);
+ path = normalize(url.substring(si + 1));
}
}
}
initDirAndName();
}
+ private String normalize(String path) {
+ // there is a slight performance penalty here, but it makes things
+ // cleaner
+ return path.replace("/./", "/");
+ }
+
private void initDirAndName() {
int di = path.lastIndexOf('/');
if (di == 0) {
@@ -126,8 +132,31 @@
}
private static final AbsFile[] FILE_ARRAY = new AbsFile[0];
+
+ public List<AbsFile> listDirectories(FilenameFilter filter) {
+ try {
+ FileResource fr = getFileResource();
+ try {
+ List<AbsFile> l = new ArrayList<AbsFile>();
+ for (GridFile gf : fr.list(path)) {
+ AbsFile f = new AbsFile(protocol, host, gf.getAbsolutePathName());
+ if (gf.isDirectory() && (filter == null || filter.accept(new File(f.getDir()), f.getName()))) {
+ l.add(f);
+ }
+ }
+ return l;
+ }
+ finally {
+ releaseResource(fr);
+ }
+ }
+ catch (Exception e) {
+ // TODO this should be a proper exception
+ throw new RuntimeException(e);
+ }
+ }
- public AbsFile[] listFiles(FilenameFilter filter) {
+ public List<AbsFile> listFiles(FilenameFilter filter) {
try {
FileResource fr = getFileResource();
try {
@@ -138,7 +167,7 @@
l.add(f);
}
}
- return l.toArray(FILE_ARRAY);
+ return l;
}
finally {
releaseResource(fr);
@@ -149,7 +178,7 @@
throw new RuntimeException(e);
}
}
-
+
public String getName() {
return name;
}
Modified: branches/faster/src/org/griphyn/vdl/mapping/AbstractMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/AbstractMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/AbstractMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -17,6 +17,9 @@
package org.griphyn.vdl.mapping;
+import java.util.HashSet;
+import java.util.Set;
+
import org.apache.log4j.Logger;
import org.griphyn.vdl.mapping.file.FileGarbageCollector;
@@ -30,8 +33,26 @@
public static final Logger logger = Logger.getLogger(AbstractMapper.class);
public static final MappingParam PARAM_INPUT = new MappingParam("input", Boolean.FALSE);
- protected MappingParamSet params;
+
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_INPUT);
+ }
+
+ protected void addParams(Set<String> s, MappingParam... params) {
+ for (MappingParam p : params) {
+ s.add(p.getName());
+ }
+ }
+
+ @Override
+ public final Set<String> getSupportedParamNames() {
+ Set<String> s = new HashSet<String>();
+ getValidMappingParams(s);
+ return s;
+ }
+ protected MappingParamSet params;
+
public synchronized void setParam(MappingParam param, Object value) {
if (params == null) {
params = new MappingParamSet();
Modified: branches/faster/src/org/griphyn/vdl/mapping/Mapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/Mapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/Mapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -18,6 +18,7 @@
package org.griphyn.vdl.mapping;
import java.util.Collection;
+import java.util.Set;
/** This interface must be implemented by a Java class that represents
a Swift mapper between SwiftScript variables and external data
@@ -78,4 +79,6 @@
boolean isPersistent(Path path);
+
+ Set<String> getSupportedParamNames();
}
Modified: branches/faster/src/org/griphyn/vdl/mapping/MapperFactory.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/MapperFactory.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/MapperFactory.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -22,6 +22,7 @@
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import org.griphyn.vdl.mapping.file.AirsnMapper;
import org.griphyn.vdl.mapping.file.ArrayFileMapper;
@@ -38,11 +39,14 @@
import org.griphyn.vdl.mapping.file.TestMapper;
public class MapperFactory {
- private static Map<String,Class<Mapper>> mappers =
- new HashMap<String,Class<Mapper>>();
+ private static Map<String, Class<? extends Mapper>> mappers;
+
+ private static Map<String, Set<String>> validParams;
static {
-
+ mappers = new HashMap<String, Class<? extends Mapper>>();
+ validParams = new HashMap<String, Set<String>>();
+
// the following are general purpose file mappers
registerMapper("simple_mapper", SimpleFileMapper.class);
registerMapper("single_file_mapper", SingleFileMapper.class);
@@ -62,11 +66,10 @@
registerMapper("test_mapper", TestMapper.class);
}
- public synchronized static Mapper getMapper(String type, MappingParamSet params)
- throws InvalidMapperException {
- Class<Mapper> cls = mappers.get(type);
+ public synchronized static Mapper getMapper(String type, MappingParamSet params) throws InvalidMapperException {
+ Class<? extends Mapper> cls = mappers.get(type);
if (cls == null) {
- throw new InvalidMapperException("no such mapper: "+type);
+ throw new InvalidMapperException("No such mapper: "+type);
}
try {
Mapper mapper = cls.newInstance();
@@ -77,15 +80,29 @@
throw new InvalidMapperException(type + ": " + e.getMessage(), e);
}
}
+
+ public synchronized static boolean isValidMapperType(String type) {
+ return mappers.containsKey(type);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static void registerMapper(String type, String cls) throws ClassNotFoundException {
+ registerMapper(type, (Class<? extends Mapper>) MapperFactory.class.getClassLoader().loadClass(cls));
+ }
- public static void registerMapper(String type, String cls)
- throws ClassNotFoundException {
- registerMapper(type, MapperFactory.class.getClassLoader().loadClass(cls));
+ public synchronized static void registerMapper(String type, Class<? extends Mapper> cls) {
+ mappers.put(type, cls);
+ try {
+ Mapper m = cls.newInstance();
+ validParams.put(type, m.getSupportedParamNames());
+ }
+ catch (Exception e) {
+ throw new RuntimeException("Cannot instantiate a '" + type + "'", e);
+ }
}
- @SuppressWarnings("unchecked")
- public synchronized static void registerMapper(String type,
- Class<?> cls) {
- mappers.put(type, (Class<Mapper>) cls);
- }
+
+ public static Set<String> getValidParams(String type) {
+ return validParams.get(type);
+ }
}
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -23,8 +23,11 @@
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.StringTokenizer;
+import java.util.Set;
import org.apache.log4j.Logger;
import org.globus.cog.util.Base64;
@@ -32,7 +35,6 @@
import org.griphyn.vdl.mapping.AbstractDataNode;
import org.griphyn.vdl.mapping.AbstractMapper;
import org.griphyn.vdl.mapping.HandleOpenException;
-import org.griphyn.vdl.mapping.InvalidMappingParameterException;
import org.griphyn.vdl.mapping.MappingParam;
import org.griphyn.vdl.mapping.MappingParamSet;
import org.griphyn.vdl.mapping.Path;
@@ -51,15 +53,12 @@
* <ul>
* <li>location - if specified, then all generated filenames
* will be prefixed with this directory, and all lookups
- * for files will happen in this directory</li>
- * <li>prefix - if specified, then all filenames will be prefixed
+ * for files will happen in this directory. If not specified,
+ * the current directory will be used.</li>
+ * <li>prefix - if specified, then all file names will be prefixed
* with this string</li>
* <li>suffix - if specified, then all filenames will be suffixed with
- * this string. If suffix does not begin with a '.'
- * character, then a '.' will be added automatically to
- * separate the rest of the filename from the suffix</li>
- * <li>noauto - if specified as "true", then the suffix auto addition of a'.'
- * will be disabled. Default value is "false".</li>
+ * this string.</li>
* <li>pattern - if specified, then filenames will be selected from
* the location directory when they match the unix glob
* pattern supplied in this parameter.</li>
@@ -71,9 +70,15 @@
public static final MappingParam PARAM_SUFFIX = new MappingParam("suffix", null);
public static final MappingParam PARAM_PATTERN = new MappingParam("pattern", null);
public static final MappingParam PARAM_LOCATION = new MappingParam("location", null);
- public static final MappingParam PARAM_NOAUTO = new MappingParam("noauto", "false");
private String location, prefix, suffix, pattern;
+
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_PREFIX, PARAM_SUFFIX, PARAM_PATTERN, PARAM_LOCATION);
+ super.getValidMappingParams(s);
+ }
public static final Logger logger = Logger.getLogger(AbstractFileMapper.class);
@@ -106,25 +111,29 @@
public void setParams(MappingParamSet params) throws HandleOpenException {
super.setParams(params);
+ StringBuilder pattern = new StringBuilder();
+ boolean wildcard = false;
+ if (PARAM_PREFIX.isPresent(this)) {
+ prefix = PARAM_PREFIX.getStringValue(this);
+ pattern.append(prefix);
+ pattern.append('*');
+ wildcard = true;
+ }
+ if (PARAM_PATTERN.isPresent(this)) {
+ pattern.append(PARAM_PATTERN.getStringValue(this));
+ wildcard = false;
+ }
if (PARAM_SUFFIX.isPresent(this)) {
- String suffix = PARAM_SUFFIX.getStringValue(this);
- String noauto = PARAM_NOAUTO.getStringValue(this);
- if (!noauto.equals("true") && !noauto.equals("false")) {
- throw new InvalidMappingParameterException("noauto parameter value should be 'true' or 'false'" +
- ". Value set was '" + noauto + "'");
+ suffix = PARAM_SUFFIX.getStringValue(this);
+ if (!wildcard) {
+ pattern.append('*');
}
- if (!suffix.startsWith(".") && noauto.equals("false")) {
- params.set(PARAM_SUFFIX, "." + suffix);
- }
+ pattern.append(suffix);
}
- if (PARAM_PATTERN.isPresent(this)) {
- String pattern = PARAM_PATTERN.getStringValue(this);
- params.set(PARAM_PATTERN, replaceWildcards(pattern));
- }
location = PARAM_LOCATION.getStringValue(this);
prefix = PARAM_PREFIX.getStringValue(this);
suffix = PARAM_SUFFIX.getStringValue(this);
- pattern = PARAM_PATTERN.getStringValue(this);
+ this.pattern = pattern.toString();
}
public PhysicalFormat map(Path path) {
@@ -219,9 +228,9 @@
}
public Collection<Path> existing() {
- if(logger.isDebugEnabled())
- logger.debug("list existing paths for mapper id="+this.hashCode());
-
+ if (logger.isDebugEnabled()) {
+ logger.debug("list existing paths for mapper id=" + this.hashCode());
+ }
List<Path> result = new ArrayList<Path>();
final AbsFile f;
if (location == null) {
@@ -229,41 +238,112 @@
}
else {
f = new AbsFile(location);
+ if (!f.exists()) {
+ throw new IllegalArgumentException("Directory not found: " + location);
+ }
}
logger.debug("Processing file list.");
- AbsFile[] files = f.listFiles(new FilenameFilter() {
- public boolean accept(File dir, String name) {
- boolean accept = (prefix == null || name.startsWith(prefix))
- && (suffix == null || name.endsWith(suffix))
- && (pattern == null || name.matches(pattern));
- logger.debug("file "+name+"? "+accept);
- return accept;
- }
- });
+ List<AbsFile> files = glob(f, pattern);
if (files != null) {
- for (int i = 0; i < files.length; i++) {
- if(logger.isDebugEnabled()) logger.debug("Processing existing file "+files[i].getName());
- Path p = rmap(files[i].getName());
+ for (AbsFile file : files) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Processing existing file " + file.getName());
+ }
+ Path p = rmap(file.getName());
if (p != null) {
- if(logger.isDebugEnabled()) logger.debug("reverse-mapped to path "+p);
+ if (logger.isDebugEnabled()) {
+ logger.debug("reverse-mapped to path " + p);
+ }
result.add(p);
- } else {
+ }
+ else {
logger.debug("reverse-mapped to nothing");
}
}
}
else {
- logger.debug("list existing paths failed for mapper id="+this.hashCode());
- throw new IllegalArgumentException("Directory not found: " + location);
+ logger.debug("No files found id=" + this.hashCode());
}
- if(logger.isDebugEnabled()) {
- logger.debug("Finish list existing paths for mapper "+this.hashCode()+" list="+result);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Finish list existing paths for mapper " + this.hashCode() + " list=" + result);
}
- // System.out.println(getVarName() + " (input): found " + result.size() + " files");
return result;
}
+
+ protected List<AbsFile> glob(AbsFile f, String pattern) {
+ if (pattern.length() == 0) {
+ pattern = "*";
+ }
+ List<AbsFile> l = new ArrayList<AbsFile>();
+ List<String> tokens;
+ StringTokenizer st = new StringTokenizer(pattern, File.separator);
+ // avoid creating an array list if only one token exists
+ String firstToken;
+ if (st.hasMoreTokens()) {
+ firstToken = st.nextToken();
+ }
+ else {
+ return Collections.emptyList();
+ }
+ if (st.hasMoreTokens()) {
+ tokens = new ArrayList<String>();
+ tokens.add(firstToken);
+ while (st.hasMoreTokens()) {
+ tokens.add(st.nextToken());
+ }
+ }
+ else {
+ tokens = Collections.singletonList(firstToken);
+ }
+ globRecursive(f, l, tokens, 0);
+ return l;
+ }
+
+ private void globRecursive(AbsFile f, List<AbsFile> l, List<String> tokens, int pos) {
+ String token = tokens.get(pos);
+ if (pos == tokens.size() - 1) {
+ if (token.equals("**")) {
+ throw new IllegalArgumentException("** cannot be the last path element in a path pattern");
+ }
+ // at the file level
+ globFiles(f, l, token);
+ }
+ else if (token.equals("**")) {
+ // recursively go through all sub-directories and match the remaining pattern tokens
+ DirectoryScanner ds = new DirectoryScanner(f);
+ while (ds.hasNext()) {
+ AbsFile dir = ds.next();
+ globRecursive(dir, l, tokens, pos + 1);
+ }
+ }
+ else {
+ // not the last path element, so a directory
+ final String regex = replaceWildcards(token);
+ List<AbsFile> dirs = f.listDirectories(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.matches(regex);
+ }
+ });
+ for (AbsFile dir : dirs) {
+ globRecursive(dir, l, tokens, pos + 1);
+ }
+ }
+ }
- private String getVarName() {
+ private void globFiles(AbsFile f, List<AbsFile> l, String token) {
+ final String regex = replaceWildcards(token);
+ List<AbsFile> files = f.listFiles(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.matches(regex);
+ }
+ });
+ l.addAll(files);
+ }
+
+
+ private String getVarName() {
AbstractDataNode var = (AbstractDataNode) getParam(MappingParam.SWIFT_HANDLE);
return var == null ? "" : var.getDisplayableName();
}
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -37,6 +37,12 @@
static Logger logger = Logger.getLogger(ArrayFileMapper.class);
public static final MappingParam PARAM_FILES = new MappingParam("files");
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_FILES);
+ super.getValidMappingParams(s);
+ }
public Collection<Path> existing() {
List<Path> l = new ArrayList<Path>();
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/CSVMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -27,6 +27,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.StringTokenizer;
import org.griphyn.vdl.mapping.AbsFile;
@@ -56,7 +57,14 @@
public static final MappingParam PARAM_HDELIMITER = new MappingParam("hdelim");
/** delimiters between content fields. default is space, tab, comma */
- public static final MappingParam PARAM_DELIMITER = new MappingParam("delim", " \t,");
+ public static final MappingParam PARAM_DELIMITER = new MappingParam("delim", " \t,");
+
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_FILE, PARAM_HEADER, PARAM_SKIP, PARAM_HDELIMITER, PARAM_DELIMITER);
+ super.getValidMappingParams(s);
+ }
/** list of column names */
private List cols = new ArrayList();
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/ConcurrentMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -36,6 +36,13 @@
public class ConcurrentMapper extends AbstractFileMapper {
public static final MappingParam PARAM_THREAD_PREFIX = new MappingParam("thread_prefix", "");
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_THREAD_PREFIX);
+ super.getValidMappingParams(s);
+ }
+
private Map<Path, PhysicalFormat> remappedPaths;
public ConcurrentMapper() {
Added: branches/faster/src/org/griphyn/vdl/mapping/file/DirectoryScanner.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/DirectoryScanner.java (rev 0)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/DirectoryScanner.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -0,0 +1,62 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Mar 17, 2013
+ */
+package org.griphyn.vdl.mapping.file;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Stack;
+
+import org.griphyn.vdl.mapping.AbsFile;
+
+public class DirectoryScanner implements Iterator<AbsFile> {
+ private Stack<Iterator<AbsFile>> paths;
+
+ public DirectoryScanner(AbsFile root) {
+ paths = new Stack<Iterator<AbsFile>>();
+ paths.push(Collections.singletonList(root).iterator());
+ }
+
+ @Override
+ public boolean hasNext() {
+ while (!paths.isEmpty() && !paths.peek().hasNext()) {
+ paths.pop();
+ }
+ if (paths.isEmpty()) {
+ return false;
+ }
+ else {
+ return true;
+ }
+ }
+
+ @Override
+ public AbsFile next() {
+ AbsFile nextdir = paths.peek().next();
+ List<AbsFile> subdirs = nextdir.listDirectories(null);
+ if (subdirs != null) {
+ paths.push(subdirs.iterator());
+ }
+ return nextdir;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public static void main(String[] args) {
+ AbsFile f = new AbsFile(".");
+ DirectoryScanner s = new DirectoryScanner(f);
+ while (s.hasNext()) {
+ System.out.println(s.next());
+ }
+ }
+}
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/ExternalMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/ExternalMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/ExternalMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -27,6 +27,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.log4j.Logger;
import org.griphyn.vdl.mapping.AbsFile;
@@ -43,7 +44,15 @@
private Map<Path, AbsFile> map;
private Map<String, Path> rmap;
- public static final MappingParam PARAM_EXEC = new MappingParam("exec");
+ public static final MappingParam PARAM_EXEC = new MappingParam("exec");
+
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_EXEC);
+ s.add("*");
+ super.getValidMappingParams(s);
+ }
private static final String[] STRING_ARRAY = new String[0];
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Set;
import java.util.StringTokenizer;
import org.griphyn.vdl.mapping.AbsFile;
@@ -36,7 +37,14 @@
an array. */
public class FixedArrayFileMapper extends AbstractMapper {
public static final MappingParam PARAM_FILES = new MappingParam("files");
-
+
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_FILES);
+ super.getValidMappingParams(s);
+ }
+
private String[] files;
public FixedArrayFileMapper() {
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -22,6 +22,7 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -40,6 +41,13 @@
public static final MappingParam PARAM_TRANSFORM = new MappingParam("transform");
private String match, source, transform;
+
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_SOURCE, PARAM_MATCH, PARAM_TRANSFORM);
+ super.getValidMappingParams(s);
+ }
public RegularExpressionMapper() {
}
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/SimpleFileMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/SimpleFileMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/SimpleFileMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -21,11 +21,19 @@
package org.griphyn.vdl.mapping.file;
import org.griphyn.vdl.mapping.HandleOpenException;
+import java.util.Set;
import org.griphyn.vdl.mapping.MappingParam;
import org.griphyn.vdl.mapping.MappingParamSet;
public class SimpleFileMapper extends AbstractFileMapper {
public static final MappingParam PARAM_PADDING = new MappingParam("padding", new Integer(4));
+
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_PADDING);
+ super.getValidMappingParams(s);
+ }
public SimpleFileMapper() {
super();
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Set;
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
@@ -33,11 +34,16 @@
*/
public class SingleFileMapper extends AbstractMapper {
- public static final MappingParam PARAM_FILE = new MappingParam("file");
+ public static final MappingParam PARAM_FILE = new MappingParam("file");
+ private AbsFile file;
- private AbsFile file;
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_FILE);
+ super.getValidMappingParams(s);
+ }
- public SingleFileMapper() {
+ public SingleFileMapper() {
super();
}
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -21,6 +21,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -44,6 +45,12 @@
public static final MappingParam PARAM_TRANSFORM = new MappingParam("transform");
private String match, transform;
+
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_SOURCE, PARAM_MATCH, PARAM_TRANSFORM);
+ super.getValidMappingParams(s);
+ }
public StructuredRegularExpressionMapper() {
}
Modified: branches/faster/src/org/griphyn/vdl/mapping/file/TestMapper.java
===================================================================
--- branches/faster/src/org/griphyn/vdl/mapping/file/TestMapper.java 2013-03-17 14:29:44 UTC (rev 6389)
+++ branches/faster/src/org/griphyn/vdl/mapping/file/TestMapper.java 2013-03-18 04:49:05 UTC (rev 6390)
@@ -19,6 +19,7 @@
import java.util.Collection;
import java.util.Collections;
+import java.util.Set;
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
@@ -36,6 +37,12 @@
public static final MappingParam PARAM_REMAPPABLE = new MappingParam("remappable", false);
public static final MappingParam PARAM_STATIC = new MappingParam("static", true);
+ @Override
+ protected void getValidMappingParams(Set<String> s) {
+ addParams(s, PARAM_FILE, PARAM_TEMP, PARAM_REMAPPABLE, PARAM_STATIC);
+ super.getValidMappingParams(s);
+ }
+
private PhysicalFormat remap, map;
private boolean remappable, temp, _static;
More information about the Swift-commit
mailing list