[Swift-commit] r7748 - in trunk: libexec src/org/griphyn/vdl/karajan/lib src/org/griphyn/vdl/mapping src/org/griphyn/vdl/mapping/file src/org/griphyn/vdl/type src/org/griphyn/vdl/type/impl
hategan at ci.uchicago.edu
hategan at ci.uchicago.edu
Thu Mar 27 20:51:51 CDT 2014
Author: hategan
Date: 2014-03-27 20:51:51 -0500 (Thu, 27 Mar 2014)
New Revision: 7748
Added:
trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java
trunk/src/org/griphyn/vdl/karajan/lib/ReadCollectList.java
trunk/src/org/griphyn/vdl/mapping/FileSystemLister.java
trunk/src/org/griphyn/vdl/mapping/file/AbstractFileNameElementMapper.java
Modified:
trunk/libexec/_swiftwrap
trunk/libexec/_swiftwrap.staging
trunk/libexec/swift-int-staging.k
trunk/libexec/swift-int.k
trunk/libexec/swift-lib.k
trunk/libexec/swift.k
trunk/src/org/griphyn/vdl/karajan/lib/AppStageouts.java
trunk/src/org/griphyn/vdl/karajan/lib/GetStagingInfo.java
trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java
trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java
trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java
trunk/src/org/griphyn/vdl/karajan/lib/RemoteFileNames.java
trunk/src/org/griphyn/vdl/mapping/AbsFile.java
trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java
trunk/src/org/griphyn/vdl/mapping/ArrayDataNode.java
trunk/src/org/griphyn/vdl/mapping/Mapper.java
trunk/src/org/griphyn/vdl/mapping/MapperFactory.java
trunk/src/org/griphyn/vdl/mapping/NullMapper.java
trunk/src/org/griphyn/vdl/mapping/RootDataNode.java
trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java
trunk/src/org/griphyn/vdl/mapping/file/AirsnFileNameElementMapper.java
trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java
trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java
trunk/src/org/griphyn/vdl/mapping/file/ConcurrentElementMapper.java
trunk/src/org/griphyn/vdl/mapping/file/DefaultFileNameElementMapper.java
trunk/src/org/griphyn/vdl/mapping/file/ExternalMapper.java
trunk/src/org/griphyn/vdl/mapping/file/FileNameElementMapper.java
trunk/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java
trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java
trunk/src/org/griphyn/vdl/mapping/file/ROIFileNameElementMapper.java
trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java
trunk/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java
trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java
trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java
trunk/src/org/griphyn/vdl/type/Type.java
trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java
trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java
Log:
support for dynamic array (and other structures) stageouts
Modified: trunk/libexec/_swiftwrap
===================================================================
--- trunk/libexec/_swiftwrap 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/libexec/_swiftwrap 2014-03-28 01:51:51 UTC (rev 7748)
@@ -305,6 +305,10 @@
OUTF=$VALUE
shift $SHIFTCOUNT
+getarg "-cf" "$@"
+COLLECT=$VALUE
+shift $SHIFTCOUNT
+
getarg "-cdmfile" "$@"
CDM_FILE=
if [ "X$VALUE" != "X" ]; then
@@ -384,7 +388,7 @@
log "DIRS=$DIRS"
log "INF=$INF"
log "OUTF=$OUTF"
-log "KICKSTART=$KICKSTART"
+log "COLLECT=$COLLECT"
log "CDM_FILE=$CDM_FILE"
log "ARGS=$@"
log "ARGC=$#"
@@ -543,6 +547,21 @@
fi
fi
+if [ "$COLLECT" != "" ]; then
+ logstate "COLLECT"
+ log "Collect list is $COLLECT"
+ exec 4<> "$WFDIR/shared/${ID}-collect"
+ for O in "$COLLECT" ; do
+ log "Collecting $O"
+ ls -1 $O >&4
+ for OO in $O ; do
+ DIR=`dirname "$OO"`
+ mv $OO "$WFDIR/shared/$DIR" 2>&1 >& "$INFO"
+ done
+ done
+ exec 4>&-
+fi
+
log "Moving back to workflow directory $WFDIR"
cd $WFDIR
if [ $? != 0 ]; then
Modified: trunk/libexec/_swiftwrap.staging
===================================================================
--- trunk/libexec/_swiftwrap.staging 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/libexec/_swiftwrap.staging 2014-03-28 01:51:51 UTC (rev 7748)
@@ -185,8 +185,8 @@
OUTF=$VALUE
shift $SHIFTCOUNT
-getarg "-k" "$@"
-KICKSTART=$VALUE
+getarg "-cf" "$@"
+COLLECT=$VALUE
shift $SHIFTCOUNT
getarg "-cdmfile" "$@"
@@ -248,7 +248,7 @@
log "DIRS=$DIRS"
log "INF=$INF"
log "OUTF=$OUTF"
-log "KICKSTART=$KICKSTART"
+log "COLLECT=$COLLECT"
log "ARGS=$@"
log "ARGC=$#"
@@ -302,6 +302,18 @@
fi
checkError $? "Application $EXEC failed with an exit code of $?" <$STDERR
+if [ "$COLLECT" != "" ]; then
+ logstate "COLLECT"
+ log "Collect list is $COLLECT"
+ exec 4<> "_collect"
+ for O in "$COLLECT" ; do
+ log "Collecting $O"
+ ls -1 $O >&4
+ done
+ exec 4>&-
+fi
+
+
logstate "EXECUTE_DONE"
log "Job ran successfully"
Modified: trunk/libexec/swift-int-staging.k
===================================================================
--- trunk/libexec/swift-int-staging.k 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/libexec/swift-int-staging.k 2014-03-28 01:51:51 UTC (rev 7748)
@@ -78,7 +78,7 @@
tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid)
cdmfile := cdm:file(),
- (localFileDirs, remoteFileDirs, inFiles, outFiles) := getStagingInfo(stagein, stageout)
+ (fileDirs, inFiles, outFiles, outCollect) := getStagingInfo(stagein, stageout)
try {
log(LOG:DEBUG, "JOB_START jobid={jobid} tr={tr}", if (arguments != null, (" arguments=", arguments)),
@@ -96,10 +96,10 @@
"-out", if(stdout == null, "stdout.txt", getFieldValue(stdout)),
"-err", if(stderr == null, "stderr.txt", getFieldValue(stderr)),
"-i", if (stdin != null, getFieldValue(stdin)),
- "-d", str:join(remoteFileDirs, "|"),
+ "-d", str:join(fileDirs, "|"),
"-if", str:join(remoteFileNames(inFiles), "|"),
"-of", str:join(remoteFileNames(outFiles), "|"),
- "-k",
+ "-cf", str:join(remoteFileNames(outCollect), "|"),
"-cdmfile", cdmfile,
"-status", "provider"
"-a", if (arguments != null, each(arguments))
@@ -139,7 +139,7 @@
}
else {
stageOut("{stdout}", "{stagingMethod}://localhost/{ddir}/{stdout}",
- mode = STAGING_MODE:ALWAYS + STAGING_MODE:IF_PRESENT)
+ mode = STAGING_MODE:IF_PRESENT)
}
if (stderr == null) {
stageOut("stderr.txt", "{stagingMethod}://localhost/{ddir}/{jobid}.stderr",
@@ -147,11 +147,15 @@
}
else {
stageOut("{stderr}", "{stagingMethod}://localhost/{ddir}/{stderr}",
- mode = STAGING_MODE:ALWAYS + STAGING_MODE:IF_PRESENT)
+ mode = STAGING_MODE:IF_PRESENT)
}
stageOut("wrapper.error", "{stagingMethod}://localhost/{ddir}/{jobid}.error",
mode = STAGING_MODE:IF_PRESENT)
- appStageouts(jobid, outFiles, stagingMethod)
+ appStageouts(jobid, outFiles, outCollect, stagingMethod)
+ if (!isEmpty(outCollect)) {
+ stageOut("_collect", "{stagingMethod}://localhost/{ddir}/{jobid}.collect",
+ mode = STAGING_MODE:ALWAYS)
+ }
if (CLEANUP_ENABLED) {
task:cleanUp(".")
@@ -159,7 +163,13 @@
)
- doRestartLog(stageout)
+ if (!isEmpty(outCollect)) {
+ readCollectList("{ddir}/{jobid}.collect")
+ }
+ else {
+ []
+ }
+
log(LOG:DEBUG, "JOB_END jobid={jobid}")
}
else catch(prev) {
Modified: trunk/libexec/swift-int.k
===================================================================
--- trunk/libexec/swift-int.k 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/libexec/swift-int.k 2014-03-28 01:51:51 UTC (rev 7748)
@@ -117,9 +117,12 @@
* to be created and (eventually) exploiting the concurrency in that.
*/
log(LOG:INFO, "START jobid={jobid} host={host} - Initializing directory structure")
- for(u, dirs) {
- cache(list(u, destdir, host)) {
- createdirs(u, destdir, host)
+ for(path, dirs) {
+ cache(list(path, destdir, host)) {
+ dc := dircat(destdir, path)
+ log(LOG:INFO, "START path={path} dir={destdir} - Creating directory structure")
+
+ dir:make(dc, host=host)
}
}
log(LOG:INFO, "END jobid={jobid} - Done initializing directory structure")
@@ -158,12 +161,8 @@
}
stageWrapperParams := function(jobid, jobdir, wrapfile, dir, host) {
- log(LOG:INFO, "START jobid={jobid} - staging in wrapper params")
- provider := provider(wrapfile)
- srchost := hostname(wrapfile)
- srcdir := swift:dirname(wrapfile)
- destdir := dircat(dir, "/parameters/{jobdir}/")
- filename := basename(wrapfile)
+ log(LOG:INFO, "START jobid={jobid} - staging in wrapper params"),
+ (provider, srchost, destdir, filename, srcdir) := splitFileURL(wrapfile, dir, destdir="parameters/{jobdir}")
cache(list(destdir, host)) {
dir:make(destdir, host=host, provider=provider)
@@ -274,6 +273,15 @@
}
log(LOG:INFO, "END jobid={jobid} - Staging out finished")
}
+
+ doStageoutCollect := function(jobid, dir, host, outFiles) {
+ fileList := "{jobid}-collect"
+ task:transfer(fileList, srchost=host, srcdir=dir)
+ collectList := readCollectList(fileList)
+ doStageout(jobid, dir, host,
+ list:join(outFiles, collectList))
+ collectList
+ }
fileSizes := function(files) {
math:sum(
@@ -359,7 +367,7 @@
stdout := if (stdout == null, "stdout.txt", getFieldValue(stdout))
stderr := if (stderr == null, "stderr.txt", getFieldValue(stderr)),
- (localFileDirs, remoteFileDirs, inFiles, outFiles) := getStagingInfo(stagein, stageout)
+ (fileDirs, inFiles, outFiles, outCollect) := getStagingInfo(stagein, stageout)
os := siteProfile(rhost, "SYSINFO:OS")
@@ -370,9 +378,10 @@
"\n-out ", stdout,
"\n-err ", stderr,
"\n-i ", if (stdin != null, getFieldValue(stdin)),
- "\n-d ", str:join(remoteFileDirs, "|"),
+ "\n-d ", str:join(fileDirs, "|"),
"\n-if ", str:join(remoteFileNames(inFiles), "|"),
"\n-of ", str:join(remoteFileNames(outFiles), "|"),
+ "\n-cf ", str:join(remoteFileNames(outCollect), "|"),
"\n-cdmfile ", cdm:file(),
"\n-status ", statusMode,
for(a, arguments) {
@@ -385,7 +394,7 @@
tmpdir := dircat("{wfdir}/jobs/{jobdir}", jobid)
try {
- createDirSet(jobid, sharedDir, rhost, localFileDirs)
+ createDirSet(jobid, sharedDir, rhost, fileDirs)
doStagein(jobid, sharedDir, rhost, inFiles)
if(wrapperMode == "files") {
stageWrapperParams(jobid, jobdir, wrapfile, wfdir, rhost)
@@ -427,9 +436,10 @@
"-out", stdout,
"-err", stderr,
"-i", if (stdin != null, getFieldValue(stdin)),
- "-d", str:join(remoteFileDirs, "|"),
+ "-d", str:join(fileDirs, "|"),
"-if", str:join(remoteFileNames(inFiles), "|"),
"-of", str:join(remoteFileNames(outFiles), "|"),
+ "-cf", str:join(remoteFileNames(outCollect), "|"),
"-cdmfile", cdm:file(),
"-status", statusMode,
"-a", if (arguments != null, each(arguments))
@@ -461,8 +471,13 @@
setProgress(progress, "Stage out")
- doStageout(jobid, sharedDir, rhost, outFiles)
- doRestartLog(stageout)
+ if (isEmpty(outCollect)) {
+ doStageout(jobid, sharedDir, rhost, outFiles)
+ [] // empty collect list
+ }
+ else {
+ doStageoutCollect(jobid, sharedDir, rhost, outFiles)
+ }
if (configProperty("wrapperlog.always.transfer") == "true") {
discard(transferWrapperLog(rhost, wfdir, jobid, jobdir))
Modified: trunk/libexec/swift-lib.k
===================================================================
--- trunk/libexec/swift-lib.k 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/libexec/swift-lib.k 2014-03-28 01:51:51 UTC (rev 7748)
@@ -131,6 +131,8 @@
export(mark, def("org.griphyn.vdl.karajan.lib.Mark"))
export(parameterlog, def("org.griphyn.vdl.karajan.lib.Parameterlog"))
export(getStagingInfo, def("org.griphyn.vdl.karajan.lib.GetStagingInfo"))
+ export(readCollectList, def("org.griphyn.vdl.karajan.lib.ReadCollectList"))
+ export(processStageouts, def("org.griphyn.vdl.karajan.lib.ProcessStageouts"))
export(doRestartLog, def("org.griphyn.vdl.karajan.lib.DoRestartLog"))
export(unwrapClosedList, def("org.griphyn.vdl.karajan.lib.UnwrapClosedList"))
Modified: trunk/libexec/swift.k
===================================================================
--- trunk/libexec/swift.k 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/libexec/swift.k 2014-03-28 01:51:51 UTC (rev 7748)
@@ -191,7 +191,7 @@
try {
throttled {
setProgress(progress, "Selecting site")
- restartOnError(number(swift:configProperty("execution.retries"))) {
+ collectList := restartOnError(number(swift:configProperty("execution.retries"))) {
if (swift:configProperty("replication.enabled") == "true") {
replicationChannel := channel:new()
//trigger the first job
@@ -235,8 +235,7 @@
}
}
}
- setDatasetValues(stageout)
- mark(stageout, false, mapping=false)
+ processStageouts(stageout, collectList)
log(LOG:INFO, "END_SUCCESS thread=", currentThread(), " tr={tr}")
setProgress(progress, "Finished successfully")
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/AppStageouts.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/AppStageouts.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/karajan/lib/AppStageouts.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -38,6 +38,7 @@
public class AppStageouts extends InternalFunction {
private ArgRef<String> jobid;
private ArgRef<List<AbsFile>> files;
+ private ArgRef<List<AbsFile>> outCollect;
private ArgRef<String> stagingMethod;
private ChannelRef<List<String>> cr_stageout;
@@ -47,7 +48,8 @@
@Override
protected Signature getSignature() {
- return new Signature(params("jobid", "files", "stagingMethod"), returns(channel("stageout")));
+ return new Signature(params("jobid", "files", "outCollect", "stagingMethod"),
+ returns(channel("stageout")));
}
@Override
@@ -60,27 +62,33 @@
try {
Stack stack = thr.getStack();
List<AbsFile> files = this.files.getValue(stack);
+ List<AbsFile> outCollect = this.outCollect.getValue(stack);
String stagingMethod = this.stagingMethod.getValue(stack);
String cwd = this.cwd.getValue(stack);
- for (AbsFile file : files) {
- String protocol = file.getProtocol();
- if (protocol.equals("file")) {
- protocol = stagingMethod;
- }
- String path = file.getDirectory() == null ? file.getName() : file.getDirectory()
- + "/" + file.getName();
- String relpath = PathUtils.remotePathName(path);
- cr_stageout.append(stack,
- makeList(relpath,
- AppStageins.localPath(cwd, protocol, path, file)));
- }
+ process(stack, files, stagingMethod, cwd);
+ process(stack, outCollect, stagingMethod, cwd);
}
catch (Exception e) {
throw new ExecutionException(this, e);
}
}
+ private void process(Stack stack, List<AbsFile> files, String stagingMethod, String cwd) {
+ for (AbsFile file : files) {
+ String protocol = file.getProtocol();
+ if (protocol.equals("file")) {
+ protocol = stagingMethod;
+ }
+ String path = file.getDirectory() == null ? file.getName() : file.getDirectory()
+ + "/" + file.getName();
+ String relpath = PathUtils.remotePathName(path);
+ cr_stageout.append(stack,
+ makeList(relpath,
+ AppStageins.localPath(cwd, protocol, path, file)));
+ }
+ }
+
private List<String> makeList(String s1, String s2) {
return new Pair<String>(s1, s2);
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/GetStagingInfo.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/GetStagingInfo.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/karajan/lib/GetStagingInfo.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -22,6 +22,7 @@
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -36,7 +37,9 @@
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.DSHandle;
import org.griphyn.vdl.mapping.HandleOpenException;
+import org.griphyn.vdl.mapping.Mapper;
import org.griphyn.vdl.type.Type;
+import org.griphyn.vdl.type.Types;
public class GetStagingInfo extends SwiftFunction {
@@ -46,51 +49,97 @@
@Override
protected Signature getSignature() {
- return new Signature(params("stageins", "stageouts"), returns(channel("...", 4)));
+ return new Signature(params("stageins", "stageouts"), returns(channel("...", 5)));
}
+
+ private static class Info {
+ Set<String> remoteDirNames = Collections.emptySet();
+ List<AbsFile> inFiles = Collections.emptyList();
+ List<AbsFile> outFiles = Collections.emptyList();
+ List<AbsFile> collectPatterns = Collections.emptyList();
+ }
-
@Override
public Object function(Stack stack) {
Collection<DSHandle> fi = stageins.getValue(stack);
Collection<DSHandle> fo = stageouts.getValue(stack);
Channel<Object> ret = cr_vargs.get(stack);
- Set<String> localDirNames = new HashSet<String>();
- Set<String> remoteDirNames = new HashSet<String>();
- List<AbsFile> inFiles = new ArrayList<AbsFile>();
- List<AbsFile> outFiles = new ArrayList<AbsFile>();
+ Info info = new Info();
+
try {
- addPaths(localDirNames, remoteDirNames, inFiles, fi);
- addPaths(localDirNames, remoteDirNames, outFiles, fo);
+ addPaths(info, fi, false);
+ addPaths(info, fo, true);
}
catch (HandleOpenException e) {
throw new ExecutionException(e.getMessage(), e);
}
- ret.add(localDirNames);
- ret.add(remoteDirNames);
- ret.add(inFiles);
- ret.add(outFiles);
+ ret.add(info.remoteDirNames);
+ ret.add(info.inFiles);
+ ret.add(info.outFiles);
+ ret.add(info.collectPatterns);
return null;
}
- private void addPaths(Set<String> ldirs, Set<String> rdirs, List<AbsFile> files, Collection<DSHandle> vars) throws HandleOpenException {
- for (DSHandle file : vars) {
- for (DSHandle leaf : file.getLeaves()) {
- Type t = leaf.getType();
- String fname = SwiftFunction.argList(SwiftFunction.filename(leaf), true);
- AbsFile f = new AbsFile(fname);
- String dir = f.getDirectory();
- if (dir != null) {
- ldirs.add(dir);
- rdirs.add(remoteDir(f, dir));
- }
- files.add(f);
+
+
+ private void addPaths(Info info, Collection<DSHandle> vars, boolean out) throws HandleOpenException {
+ for (DSHandle var : vars) {
+ Mapper m = getMapper(var);
+ if (out && !m.isStatic() && var.getType().hasArrayComponents()) {
+ Collection<AbsFile> patterns = m.getPattern(var.getPathFromRoot(), var.getType());
+ for (AbsFile f : patterns) {
+ info.collectPatterns = addOne(f, info, info.collectPatterns);
+ }
+ }
+ else {
+ addAllStatic(var, m, info, out);
+ }
+ }
+ }
+
+
+ private void addAllStatic(DSHandle var, Mapper m, Info info, boolean out) throws HandleOpenException {
+ for (DSHandle leaf : var.getLeaves()) {
+ Type t = leaf.getType();
+ if (t.equals(Types.EXTERNAL)) {
+ continue;
}
+ if (out) {
+ info.outFiles = addOne((AbsFile) m.map(leaf.getPathFromRoot()), info, info.outFiles);
+ }
+ else {
+ info.inFiles = addOne((AbsFile) m.map(leaf.getPathFromRoot()), info, info.inFiles);
+ }
}
}
+ private List<AbsFile> addOne(AbsFile f, Info info, List<AbsFile> files) {
+ String dir = f.getDirectory();
+ if (dir != null) {
+ if (info.remoteDirNames.isEmpty()) {
+ info.remoteDirNames = new HashSet<String>();
+ }
+ info.remoteDirNames.add(remoteDir(f, dir));
+ }
+ if (files.isEmpty()) {
+ files = new ArrayList<AbsFile>();
+ }
+ files.add(f);
+ return files;
+ }
+
+
+ private Mapper getMapper(DSHandle var) {
+ Mapper m = var.getMapper();
+ if (m == null) {
+ throw new ExecutionException("No mapper found for " + var);
+ }
+ return m;
+ }
+
+
private String remoteDir(AbsFile f, String dir) {
if ("file".equals(f.getProtocol())) {
return PathUtils.remotePathName(dir);
Modified: trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/karajan/lib/IsDone.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -48,22 +48,37 @@
@Override
public Object function(Stack stack) {
+ Context ctx = context.getValue(stack);
Iterable<DSHandle> files = stageout.getValue(stack);
for (DSHandle file : files) {
- if (file.isRestartable()) {
- try {
- for (DSHandle leaf : file.getLeaves()) {
- if (!IsLogged.isLogged(context.getValue(stack), leaf)) {
- return Boolean.FALSE;
+ if (file.isRestartable()) {
+ if (file.getMapper().isStatic()) {
+ try {
+ for (DSHandle leaf : file.getLeaves()) {
+ if (!IsLogged.isLogged(ctx, leaf)) {
+ return Boolean.FALSE;
+ }
}
}
- }
- catch (HandleOpenException e) {
- if (logger.isDebugEnabled()) {
- logger.debug("Handle open caught in isDone for " + file + ". Assuming false.");
+ catch (HandleOpenException e) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Handle open caught in isDone for " + file + ". Assuming false.");
+ }
+ return Boolean.FALSE;
}
- return Boolean.FALSE;
- }
+ }
+ else {
+ /*
+ * Checking if data with a dynamic mapper is done cannot involve
+ * iterating over the data items, since the items are not known
+ * until after the app runs. In such cases revert to logging
+ * and checking the entire variable rather than its items
+ */
+
+ if (!IsLogged.isLogged(ctx, file)) {
+ return Boolean.FALSE;
+ }
+ }
}
}
if (!files.iterator().hasNext()) {
Modified: trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/karajan/lib/IsLogged.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -29,6 +29,7 @@
import org.globus.cog.karajan.analyzer.ArgRef;
import org.globus.cog.karajan.analyzer.Scope;
import org.globus.cog.karajan.analyzer.Signature;
+import org.globus.cog.karajan.analyzer.Var;
import org.globus.cog.karajan.analyzer.VarRef;
import org.globus.cog.karajan.compiled.nodes.restartLog.LogEntry;
import org.globus.cog.karajan.compiled.nodes.restartLog.RestartLog;
@@ -65,6 +66,11 @@
return false;
}
+ /*
+ * One cannot use the file name to check if external data is logged
+ * since there is no explicit file name known to swift for external
+ * data.
+ */
if (var.getType().equals(Types.EXTERNAL)) {
return isLogged(logData, LogVar.getLogId(var));
}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/karajan/lib/PathUtils.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -339,11 +339,12 @@
public static class SplitFileURL extends SwiftFunction {
private ArgRef<AbsFile> file;
private ArgRef<String> dir;
+ private ArgRef<String> destdir;
private ChannelRef<Object> cr_vargs;
@Override
protected Signature getSignature() {
- return new Signature(params("file", "dir"), returns(channel("...", DYNAMIC)));
+ return new Signature(params("file", "dir", optional("destdir", null)), returns(channel("...", DYNAMIC)));
}
@@ -357,12 +358,18 @@
public Object function(Stack stack) {
AbsFile f = this.file.getValue(stack);
String dir = this.dir.getValue(stack);
+ String destdir = this.destdir.getValue(stack);
Channel<Object> ret = cr_vargs.get(stack);
ret.add(f.getProtocol());
ret.add(f.getHost());
String fdir = f.getDirectory();
- ret.add(DirCat.function(dir, RelDirName.function(fdir), false));
+ if (destdir == null) {
+ ret.add(DirCat.function(dir, RelDirName.function(fdir), false));
+ }
+ else {
+ ret.add(DirCat.function(dir, destdir, false));
+ }
ret.add(f.getName());
ret.add(fdir == null ? "" : fdir);
Added: trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/ProcessStageouts.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2012 University of Chicago
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.util.Collection;
+
+import k.rt.Channel;
+import k.rt.ExecutionException;
+import k.rt.Stack;
+
+import org.apache.log4j.Logger;
+import org.globus.cog.karajan.analyzer.ArgRef;
+import org.globus.cog.karajan.analyzer.ChannelRef;
+import org.globus.cog.karajan.analyzer.Signature;
+import org.griphyn.vdl.mapping.AbsFile;
+import org.griphyn.vdl.mapping.AbstractDataNode;
+import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.FileSystemLister;
+import org.griphyn.vdl.mapping.Mapper;
+import org.griphyn.vdl.mapping.Path;
+import org.griphyn.vdl.mapping.RootDataNode;
+
+public class ProcessStageouts extends SwiftFunction {
+ public static final Logger logger = Logger.getLogger(ProcessStageouts.class);
+
+ private ArgRef<Collection<DSHandle>> stageouts;
+ private ArgRef<Collection<AbsFile>> collectList;
+
+ private ChannelRef<Object> cr_restartLog;
+
+ @Override
+ protected Signature getSignature() {
+ return new Signature(params("stageouts", "collectList"),
+ returns(channel("restartLog", DYNAMIC)));
+ }
+
+ @Override
+ public Object function(Stack stack) {
+ Collection<DSHandle> stageouts = this.stageouts.getValue(stack);
+ Collection<AbsFile> collectList = this.collectList.getValue(stack);
+
+ if (logger.isInfoEnabled()) {
+ logger.info("Collect list: " + collectList);
+ }
+ Channel<Object> log = cr_restartLog.get(stack);
+
+ /**
+ * This basically does what setDatasetValues(), mark() and log() used to do.
+ * Specifically, discover all data in the collectList (similar to what
+ * Root*Data nodes do when initialized), set all file objects to some
+ * dummy FILE_VALUE, deep close everything, then send everything to the
+ * restart log
+ */
+ process(stageouts, collectList, log);
+ return null;
+ }
+
+ private void process(Collection<DSHandle> vars, Collection<AbsFile> collectList, Channel<Object> log) {
+ for (DSHandle var : vars) {
+ Mapper m = var.getMapper();
+ if (!m.isStatic() && var.getType().hasArrayComponents()) {
+ Collection<Path> found = m.existing(new FileSystemLister.FileList(collectList));
+ RootDataNode.addExisting(found, m, var.getRoot(), var);
+ logOnly(var, log);
+ }
+ else {
+ processStatic(var, log);
+ }
+ }
+ }
+
+
+ private void processStatic(DSHandle var, Channel<Object> log) {
+ try {
+ for (DSHandle leaf : var.getLeaves()) {
+ leaf.setValue(AbstractDataNode.FILE_VALUE);
+ leaf.closeShallow();
+ LogVar.logVar(log, leaf);
+ }
+ }
+ catch (Exception e) {
+ throw new ExecutionException("Exception caught closing stageouts", e);
+ }
+ }
+
+ private void logOnly(DSHandle var, Channel<Object> log) {
+ try {
+ for (DSHandle leaf : var.getLeaves()) {
+ LogVar.logVar(log, leaf);
+ }
+ }
+ catch (Exception e) {
+ throw new ExecutionException("Exception caught closing stageouts", e);
+ }
+ }
+}
Added: trunk/src/org/griphyn/vdl/karajan/lib/ReadCollectList.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/ReadCollectList.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/ReadCollectList.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2012 University of Chicago
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+/*
+ * Created on Jul 18, 2010
+ */
+package org.griphyn.vdl.karajan.lib;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import k.rt.ExecutionException;
+import k.rt.Stack;
+
+import org.globus.cog.karajan.analyzer.ArgRef;
+import org.globus.cog.karajan.analyzer.Param;
+import org.globus.cog.karajan.compiled.nodes.functions.AbstractSingleValuedFunction;
+import org.griphyn.vdl.mapping.AbsFile;
+
+public class ReadCollectList extends AbstractSingleValuedFunction {
+ private ArgRef<String> file;
+
+ @Override
+ protected Param[] getParams() {
+ return params("file");
+ }
+
+ @Override
+ public Object function(Stack stack) {
+ String fileName = this.file.getValue(stack);
+ try {
+ return readFile(fileName);
+ }
+ catch (IOException e) {
+ throw new ExecutionException("Could not read collect file", e);
+ }
+ }
+
+ private List<AbsFile> readFile(String fileName) throws IOException {
+ BufferedReader br = new BufferedReader(new FileReader(fileName));
+ List<AbsFile> l = new ArrayList<AbsFile>();
+
+ String line = br.readLine();
+ while (line != null) {
+ AbsFile f = new AbsFile(remoteToLocal(line));
+ l.add(f);
+ line = br.readLine();
+ }
+ br.close();
+ new File(fileName).delete();
+ return l;
+ }
+
+ /**
+ * Must do the reverse of PathUtils.remotePathName since the collect list
+ * is built remotely by _swiftwrap
+ */
+ private String remoteToLocal(String s) {
+ return s.replace("__root__/", "/").replace("__parent__", "..");
+ }
+}
Modified: trunk/src/org/griphyn/vdl/karajan/lib/RemoteFileNames.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/RemoteFileNames.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/karajan/lib/RemoteFileNames.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -21,6 +21,7 @@
package org.griphyn.vdl.karajan.lib;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import k.rt.Stack;
@@ -30,7 +31,7 @@
import org.griphyn.vdl.mapping.AbsFile;
public class RemoteFileNames extends SwiftFunction {
- private ArgRef<List<AbsFile>> files;
+ private ArgRef<Collection<AbsFile>> files;
@Override
protected Signature getSignature() {
@@ -39,7 +40,7 @@
@Override
public Object function(Stack stack) {
- List<AbsFile> files = this.files.getValue(stack);
+ Collection<AbsFile> files = this.files.getValue(stack);
List<String> ret = new ArrayList<String>();
for (AbsFile f : files) {
String path = null;
Modified: trunk/src/org/griphyn/vdl/mapping/AbsFile.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbsFile.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/AbsFile.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -36,6 +36,10 @@
public class AbsFile extends RemoteFile implements GeneralizedFileFormat {
+ public AbsFile(AbsFile dir, String name) {
+ super(dir, name);
+ }
+
public AbsFile(String url) {
super(url);
}
Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -45,7 +45,11 @@
public abstract class AbstractDataNode implements DSHandle, FutureValue {
- public static final Object FILE_VALUE = new Object();
+ public static final Object FILE_VALUE = new Object() {
+ public String toString() {
+ return "<FILE>";
+ }
+ };
static final String DATASET_URI_PREFIX = "dataset:";
Modified: trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -17,12 +17,14 @@
package org.griphyn.vdl.mapping;
+import java.util.Collection;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import org.apache.log4j.Logger;
import org.griphyn.vdl.mapping.file.FileGarbageCollector;
+import org.griphyn.vdl.type.Type;
/** AbstractMapper provides an implementation of the Mapper interface to be
used as a base class for writing other mappers. It provides handling
@@ -150,4 +152,14 @@
sb.append('>');
return sb.toString();
}
+
+ @Override
+ public Collection<AbsFile> getPattern(Path path, Type type) {
+ if (isStatic()) {
+ return null;
+ }
+ else {
+ throw new UnsupportedOperationException(this.getClass().getName() + ".getPattern()");
+ }
+ }
}
Modified: trunk/src/org/griphyn/vdl/mapping/ArrayDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/ArrayDataNode.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/ArrayDataNode.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -21,6 +21,7 @@
package org.griphyn.vdl.mapping;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
@@ -58,6 +59,21 @@
}
}
+ /**
+ * Get leaves and make sure they are sorted based on the indices
+ */
+ public void getLeaves(List<DSHandle> list) throws HandleOpenException {
+ checkMappingException();
+ if (!isClosed()) {
+ throw new HandleOpenException(this);
+ }
+ Map<Comparable<?>, DSHandle> handles = getHandles();
+ Collection<DSHandle> values = handles.values();
+ for (DSHandle child : values) {
+ ((AbstractDataNode) child).getLeaves(list);
+ }
+ }
+
/** Recursively closes arrays through a tree of arrays and complex
types. */
public void closeDeep() {
Added: trunk/src/org/griphyn/vdl/mapping/FileSystemLister.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/FileSystemLister.java (rev 0)
+++ trunk/src/org/griphyn/vdl/mapping/FileSystemLister.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -0,0 +1,180 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Mar 24, 2014
+ */
+package org.griphyn.vdl.mapping;
+
+import java.io.FilenameFilter;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.StringTokenizer;
+
+public interface FileSystemLister {
+
+ List<AbsFile> listFiles(AbsFile dir, FilenameFilter filter);
+
+ FileSystemLister DEFAULT = new Default();
+
+ public static class Default implements FileSystemLister {
+ @Override
+ public List<AbsFile> listFiles(AbsFile dir, FilenameFilter filter) {
+ return dir.listFiles(filter);
+ }
+ }
+
+ public static class FileList implements FileSystemLister {
+ private FSTree tree;
+
+ public FileList(Collection<AbsFile> l) {
+ tree = buildTree(l);
+ }
+
+ private FSTree buildTree(Collection<AbsFile> l) {
+ FSTree tree = new FSTree(null);
+ FSTree root = tree.getDir("absolute");
+ root.setParent(root);
+
+ for (AbsFile f : l) {
+ String path = f.getPath();
+ boolean absolute = path.startsWith("/");
+
+ FSTree crt = absolute ? tree.getDir("absolute") : tree.getDir("relative");
+
+ StringTokenizer st = new StringTokenizer(path, "/", false);
+ while (st.hasMoreTokens()) {
+ String token = st.nextToken();
+
+ if (!st.hasMoreTokens()) {
+ // file
+ crt.addFile(token);
+ }
+ else if (token.equals("..")) {
+ crt = crt.getParent();
+ }
+ else if (token.equals(".")) {
+ // skip
+ }
+ else {
+ crt = crt.getDir(token);
+ }
+ }
+ }
+ return tree;
+ }
+
+ @Override
+ public List<AbsFile> listFiles(AbsFile dir, FilenameFilter filter) {
+ String path = dir.getPath();
+
+ boolean absolute = path.startsWith("/");
+
+ FSTree crt = absolute ? tree.getDir("absolute") : tree.getDir("relative");
+
+ StringTokenizer st = new StringTokenizer(path, "/", false);
+ while (st.hasMoreTokens() && crt != null) {
+ String token = st.nextToken();
+
+ if (token.equals("..")) {
+ crt = crt.getParent();
+ }
+ else if (token.equals(".")) {
+ // skip
+ }
+ else {
+ crt = crt.findDir(token);
+ }
+ }
+ if (crt == null) {
+ throw new RuntimeException("No such directory: " + path);
+ }
+
+ List<AbsFile> l = new ArrayList<AbsFile>();
+ for (String name : crt.listFiles()) {
+ if (filter.accept(null, name)) {
+ l.add(new AbsFile(dir, name));
+ }
+ }
+
+ return l;
+ }
+ }
+
+ public static class FSTree {
+ private List<String> files;
+ private List<FSTreeDirEntry> dirs;
+ private FSTree parent;
+
+ public FSTree(FSTree parent) {
+ this.parent = parent;
+ }
+
+ public List<String> listFiles() {
+ if (files == null) {
+ return Collections.emptyList();
+ }
+ else {
+ return files;
+ }
+ }
+
+ public FSTree getParent() {
+ if (parent == null) {
+ parent = findDir("..");
+ }
+ return parent;
+ }
+
+ public void setParent(FSTree parent) {
+ this.parent = parent;
+ }
+
+ public void addFile(String name) {
+ if (files == null) {
+ files = new ArrayList<String>();
+ }
+ files.add(name);
+ }
+
+ public FSTree getDir(String name) {
+ FSTree d = findDir(name);
+ if (d == null) {
+ if (dirs == null) {
+ dirs = new LinkedList<FSTreeDirEntry>();
+ }
+ d = new FSTree(this);
+ dirs.add(new FSTreeDirEntry(name, d));
+ }
+ return d;
+ }
+
+ public FSTree findDir(String name) {
+ if (dirs == null) {
+ return null;
+ }
+ for (FSTreeDirEntry e : dirs) {
+ if (e.name.equals(name)) {
+ return e.files;
+ }
+ }
+ return null;
+ }
+ }
+
+ public static class FSTreeDirEntry {
+ public final String name;
+ public final FSTree files;
+
+ public FSTreeDirEntry(String name, FSTree files) {
+ this.name = name;
+ this.files = files;
+ }
+ }
+}
Modified: trunk/src/org/griphyn/vdl/mapping/Mapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/Mapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/Mapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -21,6 +21,8 @@
import java.util.Map;
import java.util.Set;
+import org.griphyn.vdl.type.Type;
+
/** This interface must be implemented by a Java class that represents
a Swift mapper between SwiftScript variables and external data
sources. */
@@ -39,6 +41,8 @@
do not necessarily have to exist.
*/
Collection<Path> existing();
+
+ Collection<Path> existing(FileSystemLister l);
/**
* Returns true if data mapped by this mapper cannot
@@ -88,4 +92,18 @@
void setBaseDir(String baseDir);
AbstractDataNode getFirstOpenParameter();
+
+ /**
+ * For dynamic mappers, this returns a glob pattern that can
+ * be used to filter data that this mapper maps from a set of
+ * files.
+ *
+ * The type parameter specifies the swift type of the data
+ * with the specified path. This is necessary in order to build
+ * a sufficiently restrictive pattern in the case when multiple
+ * levels of dynamic mappings exist (e.g. file[][]).
+ *
+ * Static mappers should return null.
+ */
+ Collection<AbsFile> getPattern(Path path, Type type);
}
Modified: trunk/src/org/griphyn/vdl/mapping/MapperFactory.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/MapperFactory.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/MapperFactory.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -117,4 +117,18 @@
public static Set<String> getValidParams(String type) {
return validParams.get(type);
}
+
+ public static void main(String[] args) {
+ for (String name : mappers.keySet()) {
+ try {
+ Mapper m = getMapper(name);
+ if (!m.isStatic()) {
+ System.out.println(name);
+ }
+ }
+ catch (InvalidMapperException e) {
+ e.printStackTrace();
+ }
+ }
+ }
}
Modified: trunk/src/org/griphyn/vdl/mapping/NullMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/NullMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/NullMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -14,7 +14,9 @@
import java.util.Map;
import java.util.Set;
+import org.griphyn.vdl.type.Type;
+
public class NullMapper implements Mapper {
public NullMapper() {
}
@@ -34,7 +36,13 @@
return Collections.emptyList();
}
+
@Override
+ public Collection<Path> existing(FileSystemLister l) {
+ return null;
+ }
+
+ @Override
public boolean isStatic() {
return false;
}
@@ -80,4 +88,9 @@
public AbstractDataNode getFirstOpenParameter() {
return null;
}
+
+ @Override
+ public Collection<AbsFile> getPattern(Path path, Type type) {
+ return null;
+ }
}
Modified: trunk/src/org/griphyn/vdl/mapping/RootDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/RootDataNode.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -20,6 +20,8 @@
*/
package org.griphyn.vdl.mapping;
+import java.util.Collection;
+
import k.rt.Future;
import k.rt.FutureListener;
import k.thr.LWThread;
@@ -156,7 +158,7 @@
static protected void checkInputs(RootHandle root, DuplicateMappingChecker dmc) {
Mapper mapper = root.getActualMapper();
if (root.isInput()) {
- addExisting(mapper, root);
+ addExisting(mapper.existing(), mapper, root, root);
checkConsistency(root, root, dmc);
}
else if (mapper.isStatic()) {
@@ -194,11 +196,11 @@
}
}
- private static void addExisting(Mapper mapper, RootHandle root) {
+ public static void addExisting(Collection<Path> existing, Mapper mapper, RootHandle root, DSHandle var) {
boolean any = false;
- for (Path p : mapper.existing()) {
+ for (Path p : existing) {
try {
- DSHandle field = root.getField(p);
+ DSHandle field = var.getField(p);
field.setValue(FILE_VALUE);
if (variableTracer.isEnabled()) {
variableTracer.trace(root.getThread(), root.getLine(),
@@ -216,7 +218,7 @@
e.getDataNode().getType() + "'");
}
}
- root.closeDeep();
+ var.closeDeep();
if (!any && variableTracer.isEnabled()) {
variableTracer.trace(root.getThread(), root.getLine(),
root.getName() + " MAPPING no files found");
Modified: trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/AbstractFileMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -19,8 +19,6 @@
import java.io.File;
import java.io.FilenameFilter;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -30,13 +28,16 @@
import java.util.StringTokenizer;
import org.apache.log4j.Logger;
-import org.globus.cog.util.Base64;
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.MappingParamSet;
import org.griphyn.vdl.mapping.Path;
import org.griphyn.vdl.mapping.PhysicalFormat;
import org.griphyn.vdl.mapping.RootHandle;
+import org.griphyn.vdl.type.Field;
+import org.griphyn.vdl.type.Type;
+import org.griphyn.vdl.type.Types;
/** A base class to build mappers which map based on filename patterns.
* It provides a large amount of default behaviour which can be
@@ -132,71 +133,51 @@
AbstractFileMapperParams cp = getParams();
return map(cp, path, cp.getPrefix());
}
+
+ protected PhysicalFormat map(AbstractFileMapperParams cp, Path path, String prefix) {
+ return map(cp, path, prefix, getElementMapper());
+ }
- protected PhysicalFormat map(AbstractFileMapperParams cp, Path path, String prefix) {
- if(logger.isDebugEnabled())
- logger.debug("mapper id="+this.hashCode()+" starting to map "+path);
- StringBuffer sb = new StringBuffer();
+ protected PhysicalFormat map(AbstractFileMapperParams cp, Path path, String prefix, FileNameElementMapper em) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("mapper id=" + this.hashCode() + " starting to map " + path);
+ }
+ StringBuilder sb = new StringBuilder();
+
String location = cp.getLocation();
- maybeAppend(sb, location);
- if (location != null && !location.endsWith("/")) {
- sb.append('/');
- }
- if (prefix != null) {
- sb.append(prefix);
- }
+ maybeAppend(sb, location);
+ if (location != null && !location.endsWith("/")) {
+ sb.append('/');
+ }
+ if (prefix != null) {
+ sb.append(prefix);
+ }
- Iterator<Path.Entry> pi = path.iterator();
- int level = 0, tokenCount = path.size();
- while (pi.hasNext()) {
- Path.Entry nextPathElement = pi.next();
- if (nextPathElement.isIndex()) {
- Comparable<?> key = nextPathElement.getKey();
- String f, token;
- if (key instanceof Integer) {
- token = key.toString();
- f = getElementMapper().mapIndex(((Integer) key).intValue());
- }
- else if (key instanceof Double) {
- token = Double.toHexString(((Double) key).doubleValue());
- f = getElementMapper().mapField(token);
- }
- else {
- MessageDigest md = getDigest();
- byte[] buf = md.digest(key.toString().getBytes());
- token = encode(buf);
- f = getElementMapper().mapField(token);
- }
- if (logger.isDebugEnabled()) {
- logger.debug("Mapping path component to " + token);
- }
- sb.append(f);
- }
- else {
- String token = (String) nextPathElement.getKey();
- if (logger.isDebugEnabled()) {
- logger.debug("Mapping path component field " + token);
- }
- String f = getElementMapper().mapField(token);
- if (logger.isDebugEnabled()) {
- logger.debug("field is mapped to: " + f);
- }
- sb.append(f);
- }
+ Iterator<Path.Entry> pi = path.iterator();
+ int level = 0, tokenCount = path.size();
+ while (pi.hasNext()) {
+ Path.Entry nextPathElement = pi.next();
+ if (nextPathElement.isIndex()) {
+ Comparable<?> key = nextPathElement.getKey();
+ String f = em.mapIndex(key, level);
+ sb.append(f);
+ }
+ else {
+ String token = (String) nextPathElement.getKey();
+ if (logger.isDebugEnabled()) {
+ logger.debug("Mapping path component field " + token);
+ }
+ String f = em.mapField(token);
+ if (logger.isDebugEnabled()) {
+ logger.debug("field is mapped to: " + f);
+ }
+ sb.append(f);
+ }
- if (!pi.hasNext()) {
- logger.debug("last element in name - not using a separator");
- }
- else if (level < tokenCount - 2) {
- logger.debug("Adding mapper-specified separator");
- sb.append(getElementMapper().getSeparator(level));
- }
- else {
- logger.debug("Adding '.' instead of mapper-specified separator");
- sb.append('.');
- }
- level++;
- }
+ appendSeparator(sb, level, tokenCount, em);
+ level++;
+ }
+
String suffix = cp.getSuffix();
if (suffix != null) {
sb.append(suffix);
@@ -208,26 +189,33 @@
return new AbsFile(sb.toString());
}
- private String encode(byte[] buf) {
- buf = Base64.encode(buf);
- char[] c = new char[buf.length];
- for (int i = 0; i < buf.length; i++) {
- c[i] = (char) buf[i];
+ protected void appendSeparator(StringBuilder sb, int level, int totalTokenCount, FileNameElementMapper em) {
+ if (level == totalTokenCount - 1) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("last element in name - not using a separator");
+ }
}
- return String.copyValueOf(c);
- }
-
- private MessageDigest getDigest() {
- try {
- return MessageDigest.getInstance("SHA-1");
+ else if (level == totalTokenCount - 2) {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Adding mapper-specified separator");
+ }
+ sb.append(em.getSeparator(level));
}
- catch (NoSuchAlgorithmException e) {
- throw new Error("JVM error: SHA-1 not available");
+ else {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Adding '.' instead of mapper-specified separator");
+ }
+ sb.append('.');
}
}
@Override
public Collection<Path> existing() {
+ return existing(FileSystemLister.DEFAULT);
+ }
+
+ @Override
+ public Collection<Path> existing(FileSystemLister fsl) {
if (logger.isDebugEnabled()) {
logger.debug("list existing paths for mapper id=" + this.hashCode());
}
@@ -246,7 +234,7 @@
}
logger.debug("Processing file list.");
String pattern = cp.getPattern();
- List<AbsFile> files = glob(f, pattern);
+ List<AbsFile> files = glob(f, fsl, pattern);
if (files != null) {
for (AbsFile file : files) {
if (logger.isDebugEnabled()) {
@@ -273,7 +261,7 @@
return result;
}
- protected List<AbsFile> glob(AbsFile f, String pattern) {
+ protected List<AbsFile> glob(AbsFile f, FileSystemLister fsl, String pattern) {
if (pattern.length() == 0) {
pattern = "*";
}
@@ -298,25 +286,25 @@
else {
tokens = Collections.singletonList(firstToken);
}
- globRecursive(f, l, tokens, 0);
+ globRecursive(f, fsl, l, tokens, 0);
return l;
}
- private void globRecursive(AbsFile f, List<AbsFile> l, List<String> tokens, int pos) {
+ private void globRecursive(AbsFile f, FileSystemLister fsl, List<AbsFile> l, List<String> tokens, int pos) {
String token = tokens.get(pos);
if (pos == tokens.size() - 1) {
if (token.equals("**")) {
throw new IllegalArgumentException("** cannot be the last path element in a path pattern");
}
// at the file level
- globFiles(f, l, token);
+ globFiles(f, fsl, l, token);
}
else if (token.equals("**")) {
// recursively go through all sub-directories and match the remaining pattern tokens
DirectoryScanner ds = new DirectoryScanner(f);
while (ds.hasNext()) {
AbsFile dir = ds.next();
- globRecursive(dir, l, tokens, pos + 1);
+ globRecursive(dir, fsl, l, tokens, pos + 1);
}
}
else {
@@ -329,14 +317,14 @@
}
});
for (AbsFile dir : dirs) {
- globRecursive(dir, l, tokens, pos + 1);
+ globRecursive(dir, fsl, l, tokens, pos + 1);
}
}
}
- private void globFiles(AbsFile f, List<AbsFile> l, String token) {
+ private void globFiles(AbsFile f, FileSystemLister fsl, List<AbsFile> l, String token) {
final String regex = replaceWildcards(token);
- List<AbsFile> files = f.listFiles(new FilenameFilter() {
+ List<AbsFile> files = fsl.listFiles(f, new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.matches(regex);
@@ -345,6 +333,7 @@
l.addAll(files);
}
+
/** Returns the SwiftScript path for a supplied filename.
*
* Splits the filename into components using the separator
@@ -425,7 +414,7 @@
/** Appends the string representation of obj to the string buffer
if obj is not null. */
- private void maybeAppend(StringBuffer sb, Object obj) {
+ private void maybeAppend(StringBuilder sb, Object obj) {
if (obj != null) {
sb.append(obj.toString());
}
@@ -455,5 +444,129 @@
public boolean isStatic() {
return false;
}
+
+ @Override
+ public Collection<AbsFile> getPattern(Path path, Type type) {
+ // it makes no sense to call this method without an array
+ if (!type.isArray()) {
+ throw new IllegalArgumentException("getPattern() must be called on an array");
+ }
+ AbstractFileMapperParams cp = getParams();
+
+ /*
+ * figure out all possible paths to leaves, then map them but
+ * replace all array indices by wildcards
+ */
+ FileNameElementMapper m = new WildcardElementMapper(getElementMapper(), path.size());
+ Path cpath = path.addLast(getTemplateIndex(type.keyType()), true);
+
+ if (!type.itemType().isComposite()) {
+ AbsFile f = (AbsFile) this.map(cp, cpath, cp.getPrefix(), m);
+ return Collections.singletonList(f);
+ }
+ else {
+ // build all possible paths to leaves
+ List<Path> l = new ArrayList<Path>();
+ traverseTypes(l, cpath, type.itemType());
+
+ List<AbsFile> rl = new ArrayList<AbsFile>();
+
+ for (Path p : l) {
+ AbsFile f = (AbsFile) this.map(cp, cpath, cp.getPrefix(), m);
+ }
+
+ return rl;
+ }
+ }
+
+ private static class WildcardElementMapper implements FileNameElementMapper {
+ private FileNameElementMapper delegate;
+ private int fixed;
+
+ public WildcardElementMapper(FileNameElementMapper delegate, int fixed) {
+ this.delegate = delegate;
+ this.fixed = fixed;
+ }
+
+ @Override
+ public String mapField(String fieldName) {
+ return delegate.mapField(fieldName);
+ }
+
+ @Override
+ public String rmapField(String pathElement) {
+ return delegate.rmapField(pathElement);
+ }
+
+ @Override
+ public String mapIndex(int index, int pos) {
+ String s = delegate.mapIndex(index, pos);
+ if (pos > fixed) {
+ return nChars(s.length());
+ }
+ else {
+ return s;
+ }
+ }
+
+ @Override
+ public String mapIndex(Object index, int pos) {
+ String s = delegate.mapIndex(index, pos);
+ if (pos >= fixed) {
+ return nChars(s.length());
+ }
+ else {
+ return s;
+ }
+ }
+
+ @Override
+ public int rmapIndex(String pathElement) {
+ return delegate.rmapIndex(pathElement);
+ }
+
+ @Override
+ public String getSeparator(int depth) {
+ return delegate.getSeparator(depth);
+ }
+
+ private String nChars(int length) {
+ char[] c = new char[length];
+ for (int i = 0; i < length; i++) {
+ c[i] = '?';
+ }
+ return new String(c);
+ }
+ }
+
+ private void traverseTypes(List<Path> l, Path path, Type t) {
+ if (!t.isComposite()) {
+ // done
+ l.add(path);
+ }
+ else if (t.isArray()) {
+ Path cpath = path.addLast(getTemplateIndex(t.keyType()), true);
+ traverseTypes(l, cpath, t.itemType());
+ }
+ else {
+ // struct
+ for (Field f : t.getFields()) {
+ Path cpath = path.addLast(f.getId());
+ traverseTypes(l, cpath, f.getType());
+ }
+ }
+ }
+
+ private Comparable<?> getTemplateIndex(Type keyType) {
+ if (keyType.equals(Types.INT)) {
+ return 1;
+ }
+ else if (keyType.equals(Types.FLOAT)) {
+ return 1.1;
+ }
+ else {
+ return "index";
+ }
+ }
}
Added: trunk/src/org/griphyn/vdl/mapping/file/AbstractFileNameElementMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/AbstractFileNameElementMapper.java (rev 0)
+++ trunk/src/org/griphyn/vdl/mapping/file/AbstractFileNameElementMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -0,0 +1,63 @@
+//----------------------------------------------------------------------
+//This code is developed as part of the Java CoG Kit project
+//The terms of the license can be found at http://www.cogkit.org/license
+//This message may not be removed or altered.
+//----------------------------------------------------------------------
+
+/*
+ * Created on Mar 22, 2014
+ */
+package org.griphyn.vdl.mapping.file;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import org.apache.log4j.Logger;
+import org.globus.cog.util.Base64;
+
+public abstract class AbstractFileNameElementMapper implements FileNameElementMapper {
+ public static final Logger logger = Logger.getLogger(AbstractFileNameElementMapper.class);
+
+ @Override
+ public String mapIndex(Object key, int pos) {
+ String token;
+ String f;
+ if (key instanceof Integer) {
+ token = key.toString();
+ f = mapIndex(((Integer) key).intValue(), pos);
+ }
+ else if (key instanceof Double) {
+ token = Double.toHexString(((Double) key).doubleValue());
+ f = mapField(token);
+ }
+ else {
+ MessageDigest md = getDigest();
+ byte[] buf = md.digest(key.toString().getBytes());
+ token = encode(buf);
+ f = mapField(token);
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("Mapping path component to " + token);
+ }
+ return f;
+ }
+
+ private String encode(byte[] buf) {
+ buf = Base64.encode(buf);
+ char[] c = new char[buf.length];
+ for (int i = 0; i < buf.length; i++) {
+ c[i] = (char) buf[i];
+ }
+ return String.copyValueOf(c);
+ }
+
+ private MessageDigest getDigest() {
+ try {
+ return MessageDigest.getInstance("SHA-1");
+ }
+ catch (NoSuchAlgorithmException e) {
+ throw new Error("JVM error: SHA-1 not available");
+ }
+ }
+
+}
Modified: trunk/src/org/griphyn/vdl/mapping/file/AirsnFileNameElementMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/AirsnFileNameElementMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/AirsnFileNameElementMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -21,7 +21,7 @@
package org.griphyn.vdl.mapping.file;
-public class AirsnFileNameElementMapper implements FileNameElementMapper {
+public class AirsnFileNameElementMapper extends AbstractFileNameElementMapper {
public static final int INDEX_WIDTH = 4;
public String mapField(String fieldName) {
@@ -35,7 +35,7 @@
return pathElement;
}
- public String mapIndex(int index) {
+ public String mapIndex(int index, int pos) {
StringBuffer sb = new StringBuffer();
sb.append('i');
String num = String.valueOf(index);
@@ -56,5 +56,4 @@
}
return "_";
}
-
}
Modified: trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/ArrayFileMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -11,6 +11,7 @@
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.InvalidPathException;
import org.griphyn.vdl.mapping.MappingParamSet;
import org.griphyn.vdl.mapping.Path;
@@ -48,6 +49,11 @@
}
@Override
+ public Collection<Path> existing(FileSystemLister l) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public PhysicalFormat map(Path path) {
if (path.isEmpty()) {
throw new IllegalArgumentException("Path cannot be empty");
Modified: trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/CSVMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -33,6 +33,7 @@
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.GeneralizedFileFormat;
import org.griphyn.vdl.mapping.InvalidMappingParameterException;
import org.griphyn.vdl.mapping.MappingParamSet;
@@ -183,9 +184,14 @@
return l;
}
- @Override
+ @Override
+ public Collection<Path> existing(FileSystemLister l) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
public boolean isStatic() {
- return false;
+ return true;
}
@Override
Modified: trunk/src/org/griphyn/vdl/mapping/file/ConcurrentElementMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/ConcurrentElementMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/ConcurrentElementMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -26,7 +26,7 @@
the same from version to version.
*/
-public class ConcurrentElementMapper implements FileNameElementMapper {
+public class ConcurrentElementMapper extends AbstractFileNameElementMapper {
/** determines how many directories and element files are permitted
in each directory. There will be no more than
@@ -43,23 +43,24 @@
return pathElement;
}
- public String mapIndex(int index) {
+ public String mapIndex(int index, int pos) {
StringBuffer sb = new StringBuffer();
sb.append("-array/");
sb.append(splitIndexByLoadFactor(index));
sb.append("/");
- sb.append("elt-"+String.valueOf(index));
+ sb.append("elt-");
+ sb.append(String.valueOf(index));
return sb.toString();
}
- String splitIndexByLoadFactor(int index)
- {
- if(index<=DIRECTORY_LOAD_FACTOR) {
+ String splitIndexByLoadFactor(int index) {
+ if (index <= DIRECTORY_LOAD_FACTOR) {
return "";
- } else {
- String prefix = "h"+String.valueOf(index % DIRECTORY_LOAD_FACTOR) +"/";
- String suffix = splitIndexByLoadFactor(index/DIRECTORY_LOAD_FACTOR);
+ }
+ else {
+ String prefix = "h" + String.valueOf(index % DIRECTORY_LOAD_FACTOR) + "/";
+ String suffix = splitIndexByLoadFactor(index / DIRECTORY_LOAD_FACTOR);
return prefix + suffix;
}
}
Modified: trunk/src/org/griphyn/vdl/mapping/file/DefaultFileNameElementMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/DefaultFileNameElementMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/DefaultFileNameElementMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -30,7 +30,7 @@
* </ul>
*/
-public class DefaultFileNameElementMapper implements FileNameElementMapper {
+public class DefaultFileNameElementMapper extends AbstractFileNameElementMapper {
public static final int DEFAULT_INDEX_WIDTH = 4;
@@ -52,7 +52,7 @@
return pathElement;
}
- public String mapIndex(int index) {
+ public String mapIndex(int index, int pos) {
StringBuilder sb = new StringBuilder(indexWidth);
String num = String.valueOf(index);
for (int i = 0; i < indexWidth - num.length(); i++) {
Modified: trunk/src/org/griphyn/vdl/mapping/file/ExternalMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/ExternalMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/ExternalMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -32,6 +32,7 @@
import org.apache.log4j.Logger;
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.MappingParamSet;
import org.griphyn.vdl.mapping.Path;
import org.griphyn.vdl.mapping.PhysicalFormat;
@@ -140,7 +141,12 @@
@Override
public Collection<Path> existing() {
return map.keySet();
- }
+ }
+
+ @Override
+ public Collection<Path> existing(FileSystemLister l) {
+ throw new UnsupportedOperationException();
+ }
public Path rmap(String name) {
if (name == null || name.equals("")) {
Modified: trunk/src/org/griphyn/vdl/mapping/file/FileNameElementMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/FileNameElementMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/FileNameElementMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -27,7 +27,9 @@
String mapField(String fieldName);
String rmapField(String pathElement);
- String mapIndex(int index);
+ String mapIndex(int index, int pos);
+ String mapIndex(Object index, int pos);
+
int rmapIndex(String pathElement);
/** Returns a string which will be used as a separator string between
Modified: trunk/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/FileSystemArrayMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -17,12 +17,15 @@
package org.griphyn.vdl.mapping.file;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.Path;
import org.griphyn.vdl.mapping.PhysicalFormat;
+import org.griphyn.vdl.type.Type;
public class FileSystemArrayMapper extends AbstractFileMapper {
private Map<Object, String> filenames = new HashMap<Object, String>();
@@ -47,7 +50,7 @@
@Override
public PhysicalFormat map(Path path) {
- if (path.size()!=1) {
+ if (path.size() != 1) {
return null;
}
if (!path.isArrayIndex(0)) {
@@ -68,5 +71,27 @@
}
}
return new AbsFile(filename);
- }
+ }
+
+ @Override
+ public Collection<AbsFile> getPattern(Path path, Type type) {
+ if (!type.isArray()) {
+ throw new IllegalArgumentException("Cannot use a non-array type with the FilesysMapper");
+ }
+ AbstractFileMapperParams cp = getParams();
+ String location = defaultValue(cp.getLocation(), "./");
+ String prefix = defaultValue(cp.getPrefix(), "");
+ String suffix = defaultValue(cp.getSuffix(), "");
+ String pattern = defaultValue(cp.getPattern(), null);
+ if (pattern != null) {
+ return Collections.singletonList(new AbsFile(location + "/" + pattern));
+ }
+ else {
+ return Collections.singletonList(new AbsFile(location + "/" + prefix + "*" + suffix));
+ }
+ }
+
+ private String defaultValue(String s, String d) {
+ return s == null ? d : s;
+ }
}
Modified: trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/FixedArrayFileMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -25,6 +25,7 @@
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.MappingParamSet;
import org.griphyn.vdl.mapping.Path;
import org.griphyn.vdl.mapping.PhysicalFormat;
@@ -86,6 +87,11 @@
}
return l;
}
+
+ @Override
+ public Collection<Path> existing(FileSystemLister l) {
+ throw new UnsupportedOperationException();
+ }
@Override
public PhysicalFormat map(Path path) {
Modified: trunk/src/org/griphyn/vdl/mapping/file/ROIFileNameElementMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/ROIFileNameElementMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/ROIFileNameElementMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -20,7 +20,7 @@
import java.util.HashMap;
import java.util.Map;
-public class ROIFileNameElementMapper implements FileNameElementMapper {
+public class ROIFileNameElementMapper extends AbstractFileNameElementMapper {
private Map<Integer, String> basenames;
int index;
@@ -46,7 +46,7 @@
return pathElement;
}
- public String mapIndex(int index) {
+ public String mapIndex(int index, int pos) {
return basenames.get(index);
}
Modified: trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/RegularExpressionMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -28,6 +28,7 @@
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.MappingParamSet;
import org.griphyn.vdl.mapping.Path;
import org.griphyn.vdl.mapping.PhysicalFormat;
@@ -73,7 +74,12 @@
else {
return Collections.emptyList();
}
- }
+ }
+
+ @Override
+ public Collection<Path> existing(FileSystemLister l) {
+ throw new UnsupportedOperationException();
+ }
public PhysicalFormat map(Path path) {
RegularExpressionMapperParams cp = getParams();
Modified: trunk/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/SingleFileMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -23,6 +23,7 @@
import java.util.Set;
import org.griphyn.vdl.mapping.AbstractMapper;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.MappingParamSet;
import org.griphyn.vdl.mapping.Path;
import org.griphyn.vdl.mapping.PhysicalFormat;
@@ -61,6 +62,11 @@
return Collections.emptyList();
}
}
+
+ @Override
+ public Collection<Path> existing(FileSystemLister l) {
+ throw new UnsupportedOperationException();
+ }
@Override
public PhysicalFormat map(Path path) {
Modified: trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/StructuredRegularExpressionMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -29,6 +29,7 @@
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.HandleOpenException;
import org.griphyn.vdl.mapping.InvalidPathException;
import org.griphyn.vdl.mapping.MappingParamSet;
@@ -84,7 +85,12 @@
}
return output;
- }
+ }
+
+ @Override
+ public Collection<Path> existing(FileSystemLister l) {
+ throw new UnsupportedOperationException();
+ }
@Override
public PhysicalFormat map(Path path) {
Modified: trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/mapping/file/TestMapper.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -23,6 +23,7 @@
import org.griphyn.vdl.mapping.AbsFile;
import org.griphyn.vdl.mapping.AbstractMapper;
+import org.griphyn.vdl.mapping.FileSystemLister;
import org.griphyn.vdl.mapping.Mapper;
import org.griphyn.vdl.mapping.MappingParamSet;
import org.griphyn.vdl.mapping.Path;
@@ -103,9 +104,19 @@
public Collection<Path> existing() {
return Collections.singletonList(Path.EMPTY_PATH);
}
+
+ @Override
+ public Collection<Path> existing(FileSystemLister l) {
+ return Collections.singletonList(Path.EMPTY_PATH);
+ }
public boolean isStatic() {
TestMapperParams cp = getParams();
- return cp.getStatic_();
+ if (cp != null) {
+ return cp.getStatic_();
+ }
+ else {
+ return false;
+ }
}
}
Modified: trunk/src/org/griphyn/vdl/type/Type.java
===================================================================
--- trunk/src/org/griphyn/vdl/type/Type.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/type/Type.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -187,5 +187,7 @@
}
- public boolean hasNonPrimitiveComponents();
+ public boolean hasNonPrimitiveComponents();
+
+ public boolean hasArrayComponents();
}
Modified: trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java
===================================================================
--- trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/type/impl/TypeImpl.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -32,7 +32,7 @@
private boolean primitive;
private Map<String, Field> fields;
private Type baseType;
- private Boolean hasNonPrimitiveComponents;
+ private Boolean hasNonPrimitiveComponents, hasArrayComponents;
public TypeImpl() {
this((URI) null, null, false);
@@ -164,6 +164,27 @@
}
return hasNonPrimitiveComponents;
}
+
+ public synchronized boolean hasArrayComponents() {
+ if (hasArrayComponents == null) {
+ if (isArray()) {
+ hasArrayComponents = true;
+ }
+ else if (isPrimitive()) {
+ hasArrayComponents = true;
+ }
+ else {
+ // struct
+ for (Field f : getFields()) {
+ if (f.getType().hasArrayComponents()) {
+ return hasArrayComponents = true;
+ }
+ }
+ hasArrayComponents = false;
+ }
+ }
+ return hasArrayComponents;
+ }
public static class Array extends TypeImpl {
Modified: trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java
===================================================================
--- trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java 2014-03-22 20:28:48 UTC (rev 7747)
+++ trunk/src/org/griphyn/vdl/type/impl/UnresolvedType.java 2014-03-28 01:51:51 UTC (rev 7748)
@@ -88,6 +88,11 @@
public boolean hasNonPrimitiveComponents() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public boolean hasArrayComponents() {
+ throw new UnsupportedOperationException();
+ }
public String getName() {
return name;
More information about the Swift-commit
mailing list