From noreply at svn.ci.uchicago.edu Wed Jul 1 10:11:01 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 1 Jul 2009 10:11:01 -0500 (CDT)
Subject: [Swift-commit] r2994 - in trunk: . docs libexec
src/org/griphyn/vdl/engine
src/org/griphyn/vdl/karajan/lib/swiftscript tests/language-behaviour
Message-ID: <20090701151101.A9D9F9CC91@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-01 10:11:01 -0500 (Wed, 01 Jul 2009)
New Revision: 2994
Added:
trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java
trunk/tests/language-behaviour/writeDataPrimitive.out.expected
trunk/tests/language-behaviour/writeDataPrimitive.swift
trunk/tests/language-behaviour/writeDataStringArray.swift
trunk/tests/language-behaviour/writeDataStringArray2.out.expected
trunk/tests/language-behaviour/writeDataStringArray2.swift
trunk/tests/language-behaviour/writeDataStruct.out.expected
trunk/tests/language-behaviour/writeDataStruct.swift
trunk/tests/language-behaviour/writeDataStructArray2.out.expected
trunk/tests/language-behaviour/writeDataStructArray2.swift
Modified:
trunk/CHANGES.txt
trunk/docs/userguide.xml
trunk/libexec/vdl-lib.xml
trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java
Log:
New procedure writeData to write data structures to a file.
This should be the inverse of readData: readData(writeData(X)) == X
and, modulo formatting and assuming data is in the correct format,
writeData(readData(X)) == X
The initial use for this is to write out long lists of filenames into a disk
file, rather than passing them on the command line.
Modified: trunk/CHANGES.txt
===================================================================
--- trunk/CHANGES.txt 2009-06-30 12:40:25 UTC (rev 2993)
+++ trunk/CHANGES.txt 2009-07-01 15:11:01 UTC (rev 2994)
@@ -1,3 +1,7 @@
+(07/01/09)
+*** New writeData procedure which writes data structures to files in the same
+ format as used by readData.
+
(06/30/09)
*** New parameter -condor-g for swift-osg-ress-site-catalog which causes
sites files to be generate which will use a local condor-g installation
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2009-06-30 12:40:25 UTC (rev 2993)
+++ trunk/docs/userguide.xml 2009-07-01 15:11:01 UTC (rev 2994)
@@ -2189,6 +2189,12 @@
+ writeData
+
+writeData will write out data structures in the format
+described for readData
+
+
Modified: trunk/libexec/vdl-lib.xml
===================================================================
--- trunk/libexec/vdl-lib.xml 2009-06-30 12:40:25 UTC (rev 2993)
+++ trunk/libexec/vdl-lib.xml 2009-07-01 15:11:01 UTC (rev 2994)
@@ -5,6 +5,7 @@
+
Modified: trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2009-06-30 12:40:25 UTC (rev 2993)
+++ trunk/src/org/griphyn/vdl/engine/ProcedureSignature.java 2009-07-01 15:11:01 UTC (rev 2994)
@@ -137,7 +137,15 @@
trace.setAnyNumOfInputArgs();
trace.setInvocationMode(INVOCATION_INTERNAL);
proceduresMap.put("trace", trace);
-
+
+ ProcedureSignature writeData = new ProcedureSignature("writeData");
+ FormalArgumentSignature wdInputArg = new FormalArgumentSignature(true);
+ writeData.addInputArg(wdInputArg);
+ FormalArgumentSignature wdOutputArg = new FormalArgumentSignature(true);
+ writeData.addOutputArg(wdOutputArg);
+ writeData.setInvocationMode(INVOCATION_INTERNAL);
+ proceduresMap.put("writeData", writeData);
+
return proceduresMap;
}
Added: trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java (rev 0)
+++ trunk/src/org/griphyn/vdl/karajan/lib/swiftscript/WriteData.java 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,179 @@
+package org.griphyn.vdl.karajan.lib.swiftscript;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+
+import org.apache.log4j.Logger;
+import org.globus.cog.karajan.arguments.Arg;
+import org.globus.cog.karajan.stack.VariableStack;
+import org.globus.cog.karajan.workflow.ExecutionException;
+import org.griphyn.vdl.karajan.lib.VDLFunction;
+import org.griphyn.vdl.mapping.AbsFile;
+import org.griphyn.vdl.mapping.AbstractDataNode;
+import org.griphyn.vdl.mapping.DSHandle;
+import org.griphyn.vdl.mapping.HandleOpenException;
+import org.griphyn.vdl.mapping.InvalidPathException;
+import org.griphyn.vdl.mapping.Path;
+import org.griphyn.vdl.mapping.PhysicalFormat;
+import org.griphyn.vdl.type.Type;
+import org.griphyn.vdl.type.Types;
+
+
+public class WriteData extends VDLFunction {
+ public static final Logger logger = Logger.getLogger(WriteData.class);
+
+ public static final Arg DEST = new Arg.Positional("dest");
+ public static final Arg SRC = new Arg.Positional("src");
+ public static boolean warning;
+
+ static {
+ setArguments(WriteData.class, new Arg[] { DEST, SRC });
+ }
+
+ protected Object function(VariableStack stack) throws ExecutionException, HandleOpenException {
+ // dest needs to be mapped to a file, or a string
+ DSHandle dest = (DSHandle) DEST.getValue(stack);
+
+ // src can be any of several forms of value
+ DSHandle src = (DSHandle) SRC.getValue(stack);
+
+ waitFor(stack, src);
+
+ if (dest.getType().equals(Types.STRING)) {
+ writeData((String)dest.getValue(), src);
+ }
+ else {
+ PhysicalFormat pf = dest.getMapper().map(Path.EMPTY_PATH);
+ if (pf instanceof AbsFile) {
+ AbsFile af = (AbsFile) pf;
+ if (!af.getProtocol().equalsIgnoreCase("file")) {
+ throw new ExecutionException("writeData only supports local files");
+ }
+ writeData(af.getPath(), src);
+ }
+ else {
+ throw new ExecutionException("writeData only supports writing to files");
+ }
+ dest.closeDeep();
+ }
+ return null;
+ }
+
+ private void writeData(String path, DSHandle src) throws ExecutionException {
+ File f = new File(path);
+ try {
+ BufferedWriter br = new BufferedWriter(new FileWriter(f));
+ try {
+ if (src.getType().isArray()) {
+ // each line is an item
+ writeArray(br, src);
+ }
+ else if (src.getType().isPrimitive()) {
+ writePrimitive(br, src);
+ }
+ else {
+ // struct
+ writeStructHeader(src.getType(), br);
+ writeStruct(br, src);
+ }
+ }
+ finally {
+ try {
+ br.close();
+ }
+ catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ catch (IOException e) {
+ throw new ExecutionException(e);
+ }
+ }
+
+ private void writePrimitive(BufferedWriter br, DSHandle src) throws IOException,
+ ExecutionException {
+ br.write(src.getValue().toString());
+ }
+
+ private void writeArray(BufferedWriter br, DSHandle src) throws IOException, ExecutionException {
+ if (src.getType().itemType().isPrimitive()) {
+ writePrimitiveArray(br, src);
+ }
+ else {
+ writeStructArray(br, src);
+ }
+ }
+
+ private void writePrimitiveArray(BufferedWriter br, DSHandle src) throws IOException,
+ ExecutionException {
+ Map m = ((AbstractDataNode) src).getArrayValue();
+ Map c = new TreeMap(new ArrayIndexComparator());
+ c.putAll(m);
+ Iterator i = c.values().iterator();
+ while(i.hasNext()) {
+ br.write(((DSHandle)i.next()).toString());
+ br.newLine();
+ }
+ }
+
+ private void writeStructArray(BufferedWriter br, DSHandle src) throws IOException,
+ ExecutionException {
+ writeStructHeader(src.getType().itemType(), br);
+ Map m = ((AbstractDataNode) src).getArrayValue();
+ Map c = new TreeMap(new ArrayIndexComparator());
+ c.putAll(m);
+ Iterator i = c.values().iterator();
+ while(i.hasNext()) {
+ writeStruct(br, (DSHandle)i.next());
+ }
+ }
+
+
+ private void writeStructHeader(Type type, BufferedWriter br) throws ExecutionException,
+ IOException {
+ List l = type.getFieldNames();
+ Iterator i = l.iterator();
+ while(i.hasNext()) {
+ br.write(i.next().toString());
+ br.write(" ");
+ }
+ br.newLine();
+ }
+
+ private void writeStruct(BufferedWriter br, DSHandle struct) throws IOException, ExecutionException {
+ List l = struct.getType().getFieldNames();
+ Iterator i = l.iterator();
+ try {
+ while(i.hasNext()) {
+ DSHandle child = struct.getField(Path.EMPTY_PATH.addLast((String)i.next()));
+ br.write(child.toString());
+ br.write(" ");
+ }
+ br.newLine();
+ } catch(InvalidPathException e) {
+ throw new ExecutionException("Unexpectedly invalid path", e);
+ }
+ }
+
+ class ArrayIndexComparator implements Comparator {
+ public int compare(Object o1, Object o2) {
+ int i1 = Integer.parseInt((String)o1);
+ int i2 = Integer.parseInt((String)o2);
+ if(i1 < i2) return -1;
+ if(i1 > i2) return 1;
+ return 0;
+ }
+ }
+}
Added: trunk/tests/language-behaviour/writeDataPrimitive.out.expected
===================================================================
--- trunk/tests/language-behaviour/writeDataPrimitive.out.expected (rev 0)
+++ trunk/tests/language-behaviour/writeDataPrimitive.out.expected 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1 @@
+foo
\ No newline at end of file
Added: trunk/tests/language-behaviour/writeDataPrimitive.swift
===================================================================
--- trunk/tests/language-behaviour/writeDataPrimitive.swift (rev 0)
+++ trunk/tests/language-behaviour/writeDataPrimitive.swift 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,8 @@
+type file;
+
+string s = "foo";
+
+file f <"writeDataPrimitive.out">;
+
+f=writeData(s);
+
Added: trunk/tests/language-behaviour/writeDataStringArray.swift
===================================================================
--- trunk/tests/language-behaviour/writeDataStringArray.swift (rev 0)
+++ trunk/tests/language-behaviour/writeDataStringArray.swift 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,8 @@
+type file;
+
+string s[] = ["foo", "bar"];
+
+file f <"writeDataStringArray.out">;
+
+f=writeData(s);
+
Added: trunk/tests/language-behaviour/writeDataStringArray2.out.expected
===================================================================
--- trunk/tests/language-behaviour/writeDataStringArray2.out.expected (rev 0)
+++ trunk/tests/language-behaviour/writeDataStringArray2.out.expected 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,5 @@
+foo
+bar
+baz
+qux
+frrrr
Added: trunk/tests/language-behaviour/writeDataStringArray2.swift
===================================================================
--- trunk/tests/language-behaviour/writeDataStringArray2.swift (rev 0)
+++ trunk/tests/language-behaviour/writeDataStringArray2.swift 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,14 @@
+type file;
+
+string s[];
+
+file f <"writeDataStringArray2.out">;
+
+f=writeData(s);
+
+s[2] = "baz";
+s[3] = "qux";
+s[0] = "foo";
+s[1] = "bar";
+s[4] = "frrrr";
+
Added: trunk/tests/language-behaviour/writeDataStruct.out.expected
===================================================================
--- trunk/tests/language-behaviour/writeDataStruct.out.expected (rev 0)
+++ trunk/tests/language-behaviour/writeDataStruct.out.expected 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,2 @@
+r c l
+Baz BAZ baz
Added: trunk/tests/language-behaviour/writeDataStruct.swift
===================================================================
--- trunk/tests/language-behaviour/writeDataStruct.swift (rev 0)
+++ trunk/tests/language-behaviour/writeDataStruct.swift 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,14 @@
+type file;
+
+type S { string l; string c; string r; }
+
+S s;
+
+file f <"writeDataStruct.out">;
+
+f=writeData(s);
+
+s.l = "baz";
+s.c = "BAZ";
+s.r = "Baz";
+
Added: trunk/tests/language-behaviour/writeDataStructArray2.out.expected
===================================================================
--- trunk/tests/language-behaviour/writeDataStructArray2.out.expected (rev 0)
+++ trunk/tests/language-behaviour/writeDataStructArray2.out.expected 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,6 @@
+r c l
+Foo FOO foo
+Bar BAR bar
+Baz BAZ baz
+Qux QUX qux
+Frrrr FRRRR frrrr
Added: trunk/tests/language-behaviour/writeDataStructArray2.swift
===================================================================
--- trunk/tests/language-behaviour/writeDataStructArray2.swift (rev 0)
+++ trunk/tests/language-behaviour/writeDataStructArray2.swift 2009-07-01 15:11:01 UTC (rev 2994)
@@ -0,0 +1,26 @@
+type file;
+
+type S { string l; string c; string r; }
+
+S s[];
+
+file f <"writeDataStructArray2.out">;
+
+f=writeData(s);
+
+s[2].l = "baz";
+s[2].c = "BAZ";
+s[2].r = "Baz";
+s[3].l = "qux";
+s[3].c = "QUX";
+s[3].r = "Qux";
+s[0].l = "foo";
+s[0].c = "FOO";
+s[0].r = "Foo";
+s[1].l = "bar";
+s[1].c = "BAR";
+s[1].r = "Bar";
+s[4].l = "frrrr";
+s[4].c = "FRRRR";
+s[4].r = "Frrrr";
+
From noreply at svn.ci.uchicago.edu Fri Jul 3 04:27:56 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 3 Jul 2009 04:27:56 -0500 (CDT)
Subject: [Swift-commit] r2995 - trunk/bin
Message-ID: <20090703092756.4D6D19CC80@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-03 04:27:55 -0500 (Fri, 03 Jul 2009)
New Revision: 2995
Modified:
trunk/bin/swift-osg-ress-site-catalog
Log:
A fix for sites having multiple gatekeepers advertised under the same site name.
Contributed by: Mats Rynge
Modified: trunk/bin/swift-osg-ress-site-catalog
===================================================================
--- trunk/bin/swift-osg-ress-site-catalog 2009-07-01 15:11:01 UTC (rev 2994)
+++ trunk/bin/swift-osg-ress-site-catalog 2009-07-03 09:27:55 UTC (rev 2995)
@@ -56,7 +56,7 @@
if ($_ eq "") {
if ($tmp{'GlueSiteName'} ne "") {
my %copy = %tmp;
- $ads{$tmp{'GlueSiteName'}} = \%copy;
+ $ads{$tmp{'GlueSiteName'} . "_" . $tmp{'GlueClusterUniqueID'}} = \%copy;
undef %tmp;
}
}
@@ -73,10 +73,10 @@
open(FH, ">$opt_out") or die("Unable to open $opt_out");
print FH "\n";
-foreach my $sitename (keys %ads) {
- my $contact = $ads{$sitename}->{'GlueCEInfoContactString'};
+foreach my $siteid (sort keys %ads) {
+ my $contact = $ads{$siteid}->{'GlueCEInfoContactString'};
my $host = $contact;
- $host =~ s/:.*//;
+ $host =~ s/[:\/].*//;
my $jm = $contact;
$jm =~ s/.*jobmanager-//;
if ($jm eq "pbs") {
@@ -91,10 +91,10 @@
elsif ($jm eq "condor") {
$jm = "Condor";
}
- my $workdir = $ads{$sitename}->{'GlueCEInfoDataDir'};
+ my $workdir = $ads{$siteid}->{'GlueCEInfoDataDir'};
print FH "\n";
- print FH " \n";
- print FH " \n";
+ print FH " \n";
+ print FH " \n";
print FH " \n";
if ($opt_condorg) {
print FH " \n";
@@ -110,7 +110,7 @@
else {
print FH " \n";
}
- print FH " $workdir/$lc_vo/tmp/$sitename\n";
+ print FH " $workdir/$lc_vo/tmp/$host\n";
print FH " \n";
}
print FH "\n\n";
From noreply at svn.ci.uchicago.edu Fri Jul 3 12:30:34 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 3 Jul 2009 12:30:34 -0500 (CDT)
Subject: [Swift-commit] r2996 - in trunk: . docs resources
src/org/griphyn/vdl/engine tests/language-behaviour
Message-ID: <20090703173034.EE0EE9CC9D@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-03 12:30:34 -0500 (Fri, 03 Jul 2009)
New Revision: 2996
Added:
trunk/tests/language-behaviour/106-import.swift
trunk/tests/language-behaviour/testimport.swift
Modified:
trunk/CHANGES.txt
trunk/docs/userguide.xml
trunk/resources/swiftscript.g
trunk/resources/swiftscript.stg
trunk/resources/swiftscript.xsd
trunk/src/org/griphyn/vdl/engine/Karajan.java
Log:
import directive to import swiftscript code from other files
Modified: trunk/CHANGES.txt
===================================================================
--- trunk/CHANGES.txt 2009-07-03 09:27:55 UTC (rev 2995)
+++ trunk/CHANGES.txt 2009-07-03 17:30:34 UTC (rev 2996)
@@ -1,3 +1,7 @@
+(07/03/09)
+*** New import keyword which allows other SwiftScript source files to be
+ imported.
+
(07/01/09)
*** New writeData procedure which writes data structures to files in the same
format as used by readData.
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2009-07-03 09:27:55 UTC (rev 2995)
+++ trunk/docs/userguide.xml 2009-07-03 17:30:34 UTC (rev 2996)
@@ -806,6 +806,37 @@
+ Imports
+
+The import directive can be used to import definitions from
+another SwiftScript file.
+
+
+For example, a SwiftScript program might contain this:
+
+import defs;
+file f;
+
+which would import the content of defs.swift in the
+current directory:
+
+type file;
+
+
+
+Imported files are read from the current working directory.
+
+
+There is no requirement that a module is imported only once. If a module
+is imported multiple times, for example in different files, then Swift will
+only process the imports once.
+
+
+Imports may contain anything that is valid in a SwiftScript program,
+including code that causes remote execution.
+
+
+
Mappers
Modified: trunk/resources/swiftscript.g
===================================================================
--- trunk/resources/swiftscript.g 2009-07-03 09:27:55 UTC (rev 2995)
+++ trunk/resources/swiftscript.g 2009-07-03 17:30:34 UTC (rev 2996)
@@ -61,6 +61,7 @@
program returns [StringTemplate code=template("program")]
:
(nsdecl[code])* //namespace declaration
+ (importStatement[code])*
(topLevelStatement[code])*
EOF
;
@@ -76,6 +77,14 @@
}
;
+importStatement [StringTemplate code]
+ : "import" name:ID SEMI {
+ StringTemplate i = template("import");
+ i.setAttribute("target", name.getText());
+ code.setAttribute("imports", i);
+ }
+ ;
+
typedecl [StringTemplate code]
{StringTemplate r=template("typeDef");
StringTemplate t=null;}
Modified: trunk/resources/swiftscript.stg
===================================================================
--- trunk/resources/swiftscript.stg 2009-07-03 09:27:55 UTC (rev 2995)
+++ trunk/resources/swiftscript.stg 2009-07-03 17:30:34 UTC (rev 2996)
@@ -1,12 +1,17 @@
group XDTM;
-program(namespaces,targetNS,functions,types,statements,sourcelocation) ::= <<
+program(namespaces,targetNS,functions,types,statements,imports,sourcelocation) ::= <<
$else$
$namespaces;separator="\n"$>$endif$
+ $if(imports)$
+
+ $imports;separator="\n"$
+
+ $endif$
$if(types)$
$types;separator="\n"$
@@ -27,6 +32,10 @@
$if(prefix)$xmlns:$prefix$="$uri$"$else$targetNamespace="$uri$"$endif$
>>
+import(target,sourcelocation) ::= <<
+$target$
+>>
+
typeDef(name,type,members,sourcelocation) ::= <<
$if(type)$
Modified: trunk/resources/swiftscript.xsd
===================================================================
--- trunk/resources/swiftscript.xsd 2009-07-03 09:27:55 UTC (rev 2995)
+++ trunk/resources/swiftscript.xsd 2009-07-03 17:30:34 UTC (rev 2996)
@@ -13,6 +13,8 @@
+
+
+
+
+
+
+
+
+
+
Modified: trunk/src/org/griphyn/vdl/engine/Karajan.java
===================================================================
--- trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-07-03 09:27:55 UTC (rev 2995)
+++ trunk/src/org/griphyn/vdl/engine/Karajan.java 2009-07-03 17:30:34 UTC (rev 2996)
@@ -1,14 +1,20 @@
package org.griphyn.vdl.engine;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
+import java.util.Set;
import javax.xml.namespace.QName;
@@ -29,6 +35,7 @@
import org.globus.swift.language.Variable.Mapping.Param;
import org.globus.swift.language.If.Else;
import org.globus.swift.language.If.Then;
+import org.globus.swift.language.ImportsDocument.Imports;
import org.globus.swift.language.ProgramDocument.Program;
import org.globus.swift.language.Switch.Case;
import org.globus.swift.language.Switch.Default;
@@ -36,6 +43,8 @@
import org.globus.swift.language.TypesDocument.Types.Type;
import org.griphyn.vdl.karajan.Loader;
import org.griphyn.vdl.karajan.CompilationException;
+import org.griphyn.vdl.toolkit.VDLt2VDLx;
+import org.griphyn.vdl.toolkit.VDLt2VDLx.ParsingException;
import org.safehaus.uuid.UUIDGenerator;
import org.w3c.dom.Node;
@@ -49,9 +58,11 @@
Map floatInternMap = new HashMap();
Map proceduresMap = new HashMap();
Map functionsMap = new HashMap();
+ Map typesMap = new HashMap();
+
+ LinkedList importList = new LinkedList();
+ Set importedNames = new HashSet();
- Types types = null;
-
int internedIDCounter = 17000;
/** an arbitrary statement identifier. Start at some high number to
@@ -139,19 +150,53 @@
return m_templates.getInstanceOf(name);
}
- public StringTemplate program(Program prog) throws CompilationException {
- VariableScope scope = new VariableScope(this, null);
- scope.bodyTemplate = template("program");
+ private void processImports(Program prog) throws CompilationException {
- scope.bodyTemplate.setAttribute("buildversion",Loader.buildVersion);
+ Imports imports = prog.getImports();
+ if(imports!=null) {
+ logger.debug("Processing SwiftScript imports");
+// process imports in reverse order
+ for(int i = imports.sizeOfImportArray() - 1 ; i >=0 ; i--) {
+ String moduleToImport = imports.getImportArray(i);
+ logger.debug("Importing module "+moduleToImport);
+ if(!importedNames.contains(moduleToImport)) {
- types = prog.getTypes();
+ // TODO PATH/PERL5LIB-style path handling
+ String swiftfilename = "./"+moduleToImport+".swift";
+ String xmlfilename = "./"+moduleToImport+".xml";
+
+
+ try {
+ VDLt2VDLx.compile(new FileInputStream(swiftfilename),new PrintStream(new FileOutputStream(xmlfilename)));
+ logger.debug("Compiled. Now reading in compiled XML for "+moduleToImport);
+ Program importedProgram = parseProgramXML(xmlfilename).getProgram();
+ logger.debug("Read in compiled XML for "+moduleToImport);
+ importList.addFirst(importedProgram);
+ importedNames.add(moduleToImport);
+ logger.debug("Added "+moduleToImport+" to import list. Processing imports from that.");
+ processImports(importedProgram);
+ } catch(Exception e) {
+ throw new CompilationException("When processing import "+moduleToImport, e);
+ }
+ } else {
+ logger.debug("Skipping repeated import of "+moduleToImport);
+ }
+ }
+ }
+ }
+
+ private void processTypes(Program prog, VariableScope scope) throws CompilationException {
+ Types types = prog.getTypes();
if (types != null) {
for (int i = 0; i < types.sizeOfTypeArray(); i++) {
Type theType = types.getTypeArray(i);
String typeName = theType.getTypename();
String typeAlias = theType.getTypealias();
-
+
+ logger.debug("Processing type "+typeName);
+
+ typesMap.put(typeName, theType);
+
StringTemplate st = template("typeDef");
st.setAttribute("name", typeName);
if (typeAlias != null && !typeAlias.equals("") && !typeAlias.equals("string")) {
@@ -172,9 +217,10 @@
scope.bodyTemplate.setAttribute("types", st);
}
}
-
- statementsForSymbols(prog, scope);
+ }
+ private void processProcedures(Program prog, VariableScope scope) throws CompilationException {
+
// Keep track of declared procedures
for (int i = 0; i < prog.sizeOfProcedureArray(); i++) {
Procedure proc = prog.getProcedureArray(i);
@@ -189,8 +235,40 @@
procedure(proc, scope);
}
- statements(prog, scope);
+ }
+
+ public StringTemplate program(Program prog) throws CompilationException {
+ VariableScope scope = new VariableScope(this, null);
+ scope.bodyTemplate = template("program");
+
+ scope.bodyTemplate.setAttribute("buildversion",Loader.buildVersion);
+
+ importList.addFirst(prog);
+ processImports(prog);
+
+ Iterator it;
+
+ it=importList.iterator();
+ while(it.hasNext()) {
+ processTypes((Program)it.next(), scope);
+ }
+
+ it=importList.iterator();
+ while(it.hasNext()) {
+ statementsForSymbols((Program)it.next(), scope);
+ }
+
+ it=importList.iterator();
+ while(it.hasNext()) {
+ processProcedures((Program)it.next(), scope);
+ }
+
+ it=importList.iterator();
+ while(it.hasNext()) {
+ statements((Program)it.next(), scope);
+ }
+
generateInternedConstants(scope.bodyTemplate);
return scope.bodyTemplate;
@@ -341,14 +419,7 @@
type = type.substring(0, type.length() - 2);
if (!type.equals("int") && !type.equals("float") && !type.equals("string")
&& !type.equals("boolean") && !type.equals("external")) {
- boolean typeDefined = false;
- if (types != null) {
- for (int i = 0; i < types.sizeOfTypeArray(); i++)
- if (types.getTypeArray(i).getTypename().equals(type)) {
- typeDefined = true;
- break;
- }
- }
+ boolean typeDefined = typesMap.containsKey(type);
if (!typeDefined)
throw new CompilationException("Type " + type + " is not defined.");
}
@@ -402,7 +473,8 @@
|| child instanceof Switch
|| child instanceof Procedure
|| child instanceof Types
- || child instanceof FormalParameter) {
+ || child instanceof FormalParameter
+ || child instanceof Imports) {
// ignore these - they're expected but we don't need to
// do anything for them here
} else {
@@ -432,7 +504,8 @@
switchStat((Switch) child, scope);
} else if (child instanceof Procedure
|| child instanceof Types
- || child instanceof FormalParameter) {
+ || child instanceof FormalParameter
+ || child instanceof Imports) {
// ignore these - they're expected but we don't need to
// do anything for them here
} else {
@@ -1073,21 +1146,19 @@
String actualType = null;
// TODO this should be a map lookup of some kind?
- for (int i = 0; i < types.sizeOfTypeArray(); i++) {
- if (types.getTypeArray(i).getTypename().equals(parentType)) {
- TypeStructure ts = types.getTypeArray(i).getTypestructure();
- int j = 0;
- for (j = 0; j < ts.sizeOfMemberArray(); j++)
- if (ts.getMemberArray(j).getMembername().equals(sm.getMemberName())) {
- actualType = ts.getMemberArray(j).getMembertype();
- break;
- }
- if (j == ts.sizeOfMemberArray())
- throw new CompilationException("No member " + sm.getMemberName()
- + " in structure " + parentType);
+
+ Type t = (Type)typesMap.get(parentType);
+
+ TypeStructure ts = t.getTypestructure();
+ int j = 0;
+ for (j = 0; j < ts.sizeOfMemberArray(); j++) {
+ if (ts.getMemberArray(j).getMembername().equals(sm.getMemberName())) {
+ actualType = ts.getMemberArray(j).getMembertype();
break;
}
- }
+ if (j == ts.sizeOfMemberArray())
+ throw new CompilationException("No member " + sm.getMemberName() + " in structure " + parentType);
+ }
if (actualType == null) {
throw new CompilationException("Type " + parentType + " is not defined.");
}
Added: trunk/tests/language-behaviour/106-import.swift
===================================================================
--- trunk/tests/language-behaviour/106-import.swift (rev 0)
+++ trunk/tests/language-behaviour/106-import.swift 2009-07-03 17:30:34 UTC (rev 2996)
@@ -0,0 +1,6 @@
+import testimport;
+
+file f;
+
+int i = 6;
+
Added: trunk/tests/language-behaviour/testimport.swift
===================================================================
--- trunk/tests/language-behaviour/testimport.swift (rev 0)
+++ trunk/tests/language-behaviour/testimport.swift 2009-07-03 17:30:34 UTC (rev 2996)
@@ -0,0 +1,4 @@
+import testimport2;
+
+type file;
+
From noreply at svn.ci.uchicago.edu Fri Jul 3 12:31:39 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 3 Jul 2009 12:31:39 -0500 (CDT)
Subject: [Swift-commit] r2997 - trunk/tests/language-behaviour
Message-ID: <20090703173139.9A8E19CC9D@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-03 12:31:39 -0500 (Fri, 03 Jul 2009)
New Revision: 2997
Added:
trunk/tests/language-behaviour/testimport2.swift
Log:
r2996 omitted a test file. this adds it.
Added: trunk/tests/language-behaviour/testimport2.swift
===================================================================
--- trunk/tests/language-behaviour/testimport2.swift (rev 0)
+++ trunk/tests/language-behaviour/testimport2.swift 2009-07-03 17:31:39 UTC (rev 2997)
@@ -0,0 +1,4 @@
+import testimport;
+
+type F;
+
From noreply at svn.ci.uchicago.edu Mon Jul 6 12:55:28 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Jul 2009 12:55:28 -0500 (CDT)
Subject: [Swift-commit] r2998 - trunk/docs
Message-ID: <20090706175528.26DD09CC9F@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-06 12:55:26 -0500 (Mon, 06 Jul 2009)
New Revision: 2998
Modified:
trunk/docs/userguide.xml
Log:
add since markers to recently documented features
Modified: trunk/docs/userguide.xml
===================================================================
--- trunk/docs/userguide.xml 2009-07-03 17:31:39 UTC (rev 2997)
+++ trunk/docs/userguide.xml 2009-07-06 17:55:26 UTC (rev 2998)
@@ -802,14 +802,14 @@
At the top level of a SwiftScript program, the global
modified may be added to a declaration so that it is visible throughout
the program, rather than only at the top level of the program. This allows
-global constants (of any type) to be defined.
+global constants (of any type) to be defined. (since Swift 0.10)
Imports
The import directive can be used to import definitions from
-another SwiftScript file.
+another SwiftScript file. (since Swift 0.10)
For example, a SwiftScript program might contain this:
@@ -1599,7 +1599,7 @@
--condor-g
Generates sites files which will submit jobs using a local Condor-G
-installation rather than through direct GRAM2 submission.
+installation rather than through direct GRAM2 submission. (since Swift 0.10)
From noreply at svn.ci.uchicago.edu Mon Jul 6 21:43:26 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Jul 2009 21:43:26 -0500 (CDT)
Subject: [Swift-commit] r2999 - SwiftApps
Message-ID: <20090707024326.3C0619CC94@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-06 21:43:25 -0500 (Mon, 06 Jul 2009)
New Revision: 2999
Added:
SwiftApps/SEE/
Log:
Added SEE app repository
From noreply at svn.ci.uchicago.edu Mon Jul 6 21:44:34 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 6 Jul 2009 21:44:34 -0500 (CDT)
Subject: [Swift-commit] r3000 - SwiftApps/SEE
Message-ID: <20090707024434.5E0F29CC94@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-06 21:44:34 -0500 (Mon, 06 Jul 2009)
New Revision: 3000
Added:
SwiftApps/SEE/trunk/
Log:
Added trunk
From noreply at svn.ci.uchicago.edu Tue Jul 7 00:02:39 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 7 Jul 2009 00:02:39 -0500 (CDT)
Subject: [Swift-commit] r3002 - SwiftApps/SEE/trunk
Message-ID: <20090707050239.85F849CC94@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-07 00:02:39 -0500 (Tue, 07 Jul 2009)
New Revision: 3002
Added:
SwiftApps/SEE/trunk/runampl.sh
SwiftApps/SEE/trunk/sites-teraport.xml
SwiftApps/SEE/trunk/sites.local.xml
SwiftApps/SEE/trunk/sites_coaster-teraport.xml
SwiftApps/SEE/trunk/swift.properties
Log:
Script for invoking swift and corresponding sites.xml entries
Added: SwiftApps/SEE/trunk/runampl.sh
===================================================================
--- SwiftApps/SEE/trunk/runampl.sh (rev 0)
+++ SwiftApps/SEE/trunk/runampl.sh 2009-07-07 05:02:39 UTC (rev 3002)
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Swift invocation scripts for various runs
+
+#swift -runid localrun -sites.file ./sites.local.xml \
+ #-tc.file ./tc.data ampl.swift
+#swift -runid teraport_coaster -sites.file ./sites_coaster-teraport.xml \
+ #-tc.file ./tc.data ampl.swift
+#swift -runid teraport -sites.file ./sites-teraport.xml \
+ #-tc.file ./tc.data ampl.swift
+#swift -runid firefly_run -config swift.properties \
+ #-sites.file ./ff-grid.xml -tc.file ./tc.data ampl.swift
Property changes on: SwiftApps/SEE/trunk/runampl.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: SwiftApps/SEE/trunk/sites-teraport.xml
===================================================================
--- SwiftApps/SEE/trunk/sites-teraport.xml (rev 0)
+++ SwiftApps/SEE/trunk/sites-teraport.xml 2009-07-07 05:02:39 UTC (rev 3002)
@@ -0,0 +1,13 @@
+
+
+
+
+ /gpfs/teraport/aespinosa/amplr-runs
+
+ 500
+ 1.98
+
+ long
+ 08:00:00
+
+
Added: SwiftApps/SEE/trunk/sites.local.xml
===================================================================
--- SwiftApps/SEE/trunk/sites.local.xml (rev 0)
+++ SwiftApps/SEE/trunk/sites.local.xml 2009-07-07 05:02:39 UTC (rev 3002)
@@ -0,0 +1,10 @@
+
+
+
+
+
+ /home/aespinosa/science/see/test_run
+ 5
+
+
+
Added: SwiftApps/SEE/trunk/sites_coaster-teraport.xml
===================================================================
--- SwiftApps/SEE/trunk/sites_coaster-teraport.xml (rev 0)
+++ SwiftApps/SEE/trunk/sites_coaster-teraport.xml 2009-07-07 05:02:39 UTC (rev 3002)
@@ -0,0 +1,20 @@
+
+
+
+
+
+ /home/aespinosa/work/ampl
+
+ 0.5
+ 1.98
+
+ short
+ 4
+ 2
+ 1
+ 8
+ 240
+ 00:24:00
+
+
Added: SwiftApps/SEE/trunk/swift.properties
===================================================================
--- SwiftApps/SEE/trunk/swift.properties (rev 0)
+++ SwiftApps/SEE/trunk/swift.properties 2009-07-07 05:02:39 UTC (rev 3002)
@@ -0,0 +1,329 @@
+sites.file=${swift.home}/etc/sites.xml
+tc.file=${swift.home}/etc/tc.data
+
+#
+# The host name of the submit machine is used by GRAM as a callback
+# address to report the status of submitted jobs. In general, Swift
+# can automatically detect the host name of the local machine.
+# However, if the machine host name is improperly configured or if
+# it does not represent a valid DNS entry, certain services (such as
+# GRAM) will not be able to send job status notifications back to
+# the client. The value of this property can be an IP address.
+#
+# Format:
+# hostname=string
+#
+
+
+#hostname=localhost
+
+#
+# A TCP port range can be specified to restrict the ports on which GRAM
+# callback services are started. This is likely needed if your submit
+# host is behind a firewall, in which case the firewall should be
+# configured to allow incoming connections on ports in the range.
+#
+# Format:
+# tcp.port.range=start,end
+#
+
+#tcp.port.range=50000,50100
+
+#
+# false - means an error will be immediately reported and cause the
+# workflow to abort. At this time remote jobs that are already
+# running will not be canceled
+# true - means that Swift will try to do as much work as possible and
+# report all errors encountered at the end. However, "errors"
+# here only applies to job execution errors. Certain errors
+# that are related to the Swift implementation (should such
+# errors occur) will still be reported eagerly.
+#
+# Default: false
+#
+lazy.errors=false
+
+#
+# What algorithm to use for caching of remote files. LRU (as in what
+# files to purge) is the only implementation right now. One can set
+# a target size (in bytes) for a host by using the swift:storagesize
+# profile for a host in sites.xml
+#
+# Default: LRU
+#
+caching.algorithm=LRU
+
+#
+# true - generate a provenance graph in .dot format (Swift will
+# choose a random file name)
+# false - do not generate a provenance graph
+# - generate a provenange graph in the give file name
+#
+# Default: false
+#
+pgraph=false
+
+
+#
+# graph properties for the provenance graph (.dot specific)
+#
+# Default: splines="compound", rankdir="TB"
+#
+pgraph.graph.options=splines="compound", rankdir="TB"
+
+
+#
+# node properties for the provenance graph (.dot specific)
+#
+# Default: color="seagreen", style="filled"
+#
+pgraph.node.options=color="seagreen", style="filled"
+
+#
+# true - clustering of small jobs is enabled. Clustering works in the
+# following way: If a job is clusterable (meaning that it has the
+# GLOBUS::maxwalltime profile specified in tc.data and its value
+# is less than the value of the "clustering.min.time" property) it will
+# be put in a clustering queue. The queue is processed at intervals
+# specified by the "clustering.queue.delay" property. The processing
+# of the clustering queue consists of selecting compatible jobs and
+# grouping them in clusters whose max wall time does not exceed twice
+# the value of the "clustering.min.time" property. Two or more jobs are
+# considered compatible if they share the same site and do not have
+# conflicting profiles (e.g. different values for the same environment
+# variable).
+# false - clustering of small jobs is disabled.
+#
+# Default: false
+#
+clustering.enabled=false
+
+
+#
+# - the intervals at which the clustering queue is processed
+#
+# Default: 4
+#
+clustering.queue.delay=4
+
+#
+# - the threshold time for clustering
+#
+# Default: 60
+#
+clustering.min.time=60
+
+#
+# Kickstart is a useful tool that can be used to gather various information
+# about a remote process. Before it can be used it must be installed on the
+# remote site and the corresponding entry be set in the sites file.
+# This option allows controlling of how Swift uses Kickstart. The following
+# values are possible:
+# false - do not use Kickstart
+# true - use Kickstart. If a job is scheduled on a site that does not have
+# Kickstart installed, that job will fail.
+# maybe - Use Kickstart if installed (i.e. the entry is present in the sites
+# file)
+#
+# Default: maybe
+#
+
+kickstart.enabled=maybe
+
+#
+# Indicates when Kickstart records should be fetched from the remote site:
+# true - always transfer Kickstart records if Kickstart was used (see
+# kickstart.enabled)
+# false - only transfer Kickstart records if the job fails
+#
+# Default: false
+#
+
+kickstart.always.transfer=false
+
+#
+# Indicates when wrapper logs should be fetched from the remote site:
+# true - always transfer wrapper logs
+# false - only transfer wrapper logs if the job fails
+#
+# Default: false
+#
+
+wrapperlog.always.transfer=false
+
+###########################################################################
+# Throttling options #
+###########################################################################
+#
+# For the throttling parameters, valid values are either a positive integer
+# or "off" (without the quotes).
+#
+
+#
+# Limits the number of concurrent submissions for a workflow instance. This
+# throttle only limits the number of concurrent tasks (jobs) that are being
+# sent to sites, not the total number of concurrent jobs that can be run.
+# The submission stage in GRAM is one of the most CPU expensive stages (due
+# mostly to the mutual authentication and delegation). Having too many
+# concurrent submissions can overload either or both the submit host CPU
+# and the remote host/head node causing degraded performance.
+#
+# Default: 4
+#
+
+throttle.submit=4
+#throttle.submit=off
+
+#
+# Limits the number of concurrent submissions for any of the sites Swift will
+# try to send jobs to. In other words it guarantees that no more than the
+# value of this throttle jobs sent to any site will be concurrently in a state
+# of being submitted.
+#
+# Default: 2
+#
+
+throttle.host.submit=2
+#throttle.host.submit=off
+
+#
+# The Swift scheduler has the ability to limit the number of concurrent jobs
+# allowed on a site based on the performance history of that site. Each site
+# is assigned a score (initially 1), which can increase or decrease based
+# on whether the site yields successful or faulty job runs. The score for a
+# site can take values in the (0.1, 100) interval. The number of allowed jobs
+# is calculated using the following formula:
+# 2 + score*throttle.score.job.factor
+# This means a site will always be allowed at least two concurrent jobs and
+# at most 2 + 100*throttle.score.job.factor. With a default of 4 this means
+# at least 2 jobs and at most 402.
+#
+# Default: 4
+#
+
+throttle.score.job.factor=0.2
+#throttle.score.job.factor=off
+
+
+#
+# Limits the total number of concurrent file transfers that can happen at any
+# given time. File transfers consume bandwidth. Too many concurrent transfers
+# can cause the network to be overloaded preventing various other signalling
+# traffic from flowing properly.
+#
+# Default: 4
+#
+
+throttle.transfers=4
+#throttle.transfers=off
+
+# Limits the total number of concurrent file operations that can happen at any
+# given time. File operations (like transfers) require an exclusive connection
+# to a site. These connections can be expensive to establish. A large number
+# of concurrent file operations may cause Swift to attempt to establish many
+# such expensive connections to various sites. Limiting the number of concurrent
+# file operations causes Swift to use a small number of cached connections and
+# achieve better overall performance.
+#
+# Default: 8
+#
+
+throttle.file.operations=8
+#throttle.file.operations=off
+
+# Indicates whether the working directory on the remote site should be
+# left intact even when the workflow completes successfully. This can be
+# used to inspect the site working directory for debugging purposes.
+#
+# Default: false
+#
+
+sitedir.keep=true
+
+# number of time a job will be retried if it fails (giving a maximum of
+# 1 + execution.retries attempts at execution)
+#
+
+execution.retries=1
+
+
+# Enables/disables replication. Replication is used to deal with jobs sitting
+# in batch queues for abnormally large amounts of time. If replication is enabled
+# and certain conditions are met, Swift creates and submits replicas of jobs, and
+# allows multiple instances of a job to compete.
+#
+
+replication.enabled=false
+
+# If replication is enabled, this value specifies the minimum time, in seconds,
+# a job needs to be queued in a batch queue in order to be considered for
+# replication
+#
+
+replication.min.queue.time=60
+
+# The maximum number of replicas that Swift should attempt.
+
+replication.limit=3
+
+#
+# WARNING: This option is deprecated. Please use the hostname option.
+#
+# The IP address of the submit machine is used by GRAM as a callback
+# address to report the status of submitted jobs. In general, Swift
+# can automatically detect the IP address of the local machine.
+# However, if the machine has more than one network interface, Swift
+# will pick the first one, which may not be the right choice. It is
+# recommended that this property is set properly before attempting to
+# run jobs through GRAM.
+#
+# Format:
+# ip.address=x.y.z.w
+#
+
+#ip.address=127.0.0.1
+
+
+# Controls how Swift will communicate the result code of running user programs
+# from workers to the submit side. In files mode, a file
+# indicating success or failure will be created on the site shared filesystem.
+# In provider mode, the execution provider job status will
+# be used. Notably, GRAM2 does not return job statuses correctly, and so
+# provider mode will not work with GRAM2. With other
+# providers, it can be used to reduce the amount of filesystem access compared
+# to files mode.
+#
+# status.mode=files
+
+# Controls how swift will supply parameters to the remote wrapper script.
+# 'args' mode will pass parameters on the command line
+# 'files' mode will pass parameters through an additional input file
+#
+# valid values: args, files
+# Default: files
+#
+# wrapper.parameter.mode=args
+
+# Determines if Swift remote wrappers will be executed by specifying an
+# absolute path, or a path relative to the job initial working directory
+#
+# valid values: absolute, relative
+# wrapper.invocation.mode=absolute
+
+#
+# Limits the number of concurrent iterations that each foreach statement
+# can have at one time. This conserves memory for swift programs that
+# have large numbers of iterations (which would otherwise all be executed
+# in parallel).
+#
+# Default: 1024
+#
+
+foreach.max.threads=1024
+
+# controls whether the log file will contain provenance information
+# enabling this will increase the size of log files, sometimes
+# significantly.
+
+provenance.log=false
+
From noreply at svn.ci.uchicago.edu Tue Jul 7 16:41:30 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 7 Jul 2009 16:41:30 -0500 (CDT)
Subject: [Swift-commit] r3003 - in SwiftApps/SEE/trunk: . subproblems
Message-ID: <20090707214130.32C179CC9F@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-07 16:41:29 -0500 (Tue, 07 Jul 2009)
New Revision: 3003
Added:
SwiftApps/SEE/trunk/subproblems/
SwiftApps/SEE/trunk/subproblems/producer_graph.mod
SwiftApps/SEE/trunk/subproblems/producer_tree.mod
Log:
Added subproblems directory from orig tarballs
Added: SwiftApps/SEE/trunk/subproblems/producer_graph.mod
===================================================================
--- SwiftApps/SEE/trunk/subproblems/producer_graph.mod (rev 0)
+++ SwiftApps/SEE/trunk/subproblems/producer_graph.mod 2009-07-07 21:41:29 UTC (rev 3003)
@@ -0,0 +1,194 @@
+##### Producer Variables #####
+#
+# Producers_Input: quantity of the input commodities used
+# Producers_Output: quantity of the output commodities produced
+#
+# Producers_Functions_Input: quantity of the inputs used by the production function
+# Producers_Functions_Output: quantity of the outputs produced by the production function
+#
+##### Notes #####
+#
+# (1) Input and Output are the primary optimization variables
+# (2) Functions_Input and Functions_Output are internal variables
+#
+##### Example #####
+#
+# Input[p1, usa, usa, agr];
+# Input[p1, usa, eur, agr];
+# Input[p1, usa, usa, lab];
+# Input[p1, usa, usa, cap];
+# Output[p1, usa, usa, agi];
+#
+# FInput[p1, usa, f1, usa, lab];
+# FInput[p1, usa, f1, usa, cap];
+# FOutput[p1, usa, f1, usa, kla];
+#
+# FInput[p1, usa, f2, usa, agr];
+# FInput[p1, usa, f2, eur, agr];
+# FInput[p1, usa, f2, usa, kla];
+# FOutput[p1, usa, f2, usa, agi];
+#
+# Input[p1, eur, usa, agr];
+# Input[p1, eur, eur, agr];
+# Input[p1, eur, eur, lab];
+# Input[p1, eur, eur, cap];
+# Output[p1, eur, eur, agi];
+#
+# FInput[p1, eur, f1, eur, lab];
+# FInput[p1, eur, f1, eur, cap];
+# FOutput[p1, eur, f1, eur, kla];
+#
+# FInput[p1, eur, f2, usa, agr];
+# FInput[p1, eur, f2, eur, agr];
+# FInput[p1, eur, f2, eur, kla];
+# FOutput[p1, eur, f2, eur, agi];
+#
+# Input[p1, row, usa, agr];
+# Input[p1, row, eur, agr];
+# Input[p1, row, row, agr];
+# Input[p1, row, row, lab];
+# Input[p1, row, row, cap];
+# Output[p1, row, eur, agi];
+#
+# FInput[p1, row, f1, row, lab];
+# FInput[p1, row, f1, row, cap];
+# FOutput[p1, row, f1, row, kla];
+#
+# FInput[p1, row, f2, usa, agr];
+# FInput[p1, row, f2, eur, agr];
+# FInput[p1, row, f2, row, agr];
+# FInput[p1, row, f2, row, kla];
+# FOutput[p1, row, f2, row, agi];
+#
+#####
+
+var Producers_Input {p in Producers, r in Producers_Regions[p], Producers_Inputs[p,r]} := 1.0;
+var Producers_Output {p in Producers, r in Producers_Regions[p], Producers_Outputs[p,r]} := 1.0;
+
+var Producers_Functions_Input {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], Producers_Functions_Inputs[p,r,f]} := 1.0;
+var Producers_Functions_Output {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], Producers_Functions_Outputs[p,r,f]} := 1.0;
+
+##### Producer Optimization Problem #####
+#
+# maximize Producers_Profits {p in Producers, r in Producers_Regions[p]}:
+# (sum {(ro,ko) in Producers_Outputs[p,r]} ((Producers_Revenues[p,r,ro,ko] -
+# Producers_RRPriceTax_Revenues[p,r,ro,ko]*Producers_RRPriceRate_Revenues[p,r,ro,ko] -
+# Producers_ROPriceTax_Revenues[p,r,ro,ko]*Producers_ROPriceRate_Revenues[p,r,ro,ko]
+# )*(if ko in Homogenous_Commodities the HPrice[ko] else Price[ro,ko]) -
+# Producers_RRQuantityTax_Revenues[p,r,ro,ko]*Producers_RRQuantityRate_Revenues[p,r,ro,ko] -
+# Producers_ROQuantityTax_Revenues[p,r,ro,ko]*Producers_ROQuantityRate_Revenues[p,r,ro,ko]
+# )*Producers_Output[p,r,ro,ko] -
+# sum {(ri,ki) in Producers_Inputs[p,r]} ((Producers_Expenditures[p,r,ri,ki] +
+# Producers_RRPriceTax_Expenditures[p,r,ri,ki]*Producers_RRPriceRate_Expenditures[p,r,ri,ki] +
+# Producers_RIPriceTax_Expenditures[p,r,ri,ki]*Producers_RIPriceRate_Expenditures[p,r,ri,ki]
+# )*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) +
+# Producers_RRQuantityTax_Expenditures[p,r,ri,ki]*Producers_RRQuantityRate_Expenditures[p,r,ri,ki] +
+# Producers_RIQuantityTax_Expenditures[p,r,ri,ki]*Producers_RIQuantityRate_Expenditures[p,r,ri,ki]
+# )*Producers_Input[p,r,ri,ki]
+# )/Producers_Scale[p,r];
+#
+# Producers_Functions_Positive_Def {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Positive[p,r,f]}:
+# Producers_Functions_Output[p,r,f,ro,ko] <= cesp(Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko], Producers_Functions_Sigma[p,r,f,ro,ko], {(ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} (Producers_Factors_Index[p,r,ri,ki], Producers_Functions_Share[p,r,f,ro,ko,ri,ki], Producers_Functions_Input[p,r,f,ri,ki]));
+#
+# Producers_Functions_Leontief_Def {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Leontief[p,r,f], (ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0}:
+# Producers_Functions_Output[p,r,f,ro,ko] <= Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko]*Producers_Functions_Input[p,r,f,ri,ki];
+#
+# Producers_Inputs_Balance_Def {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Inputs[p,r]}:
+# Producers_Input[p,r,rf,kf] >= sum {f in Producers_Functions_Regions[p,r]: (rf,kf) in Producers_Functions_Inputs[p,r,f]} Producers_Functions_Input[p,r,f,rf,kf];
+#
+# Producers_Outputs_Balance_Def {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Outputs[p,r]}:
+# Producers_Output[p,r,rf,kf] <= sum {f in Producers_Functions_Regions[p,r]: (rf,kf) in Producers_Functions_Outputs[p,r,f]} Producers_Functions_Output[p,r,f,rf,kf];
+#
+# Producers_Intermeds_Balance_Def {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Intermeds[p,r]}:
+# sum {f in Producers_Functions_Regions[p,r]: (rf,kf) in Producers_Functions_Outputs[p,r,f]} Producers_Functions_Output[p,r,f,rf,kf] >= sum {f in Producers_Functions_Regions[p,r]: (rf,kf) in Producers_Functions_Inputs[p,r,f]} Producers_Functions_Input[p,r,f,rf,kf] ;
+#
+##### Notes #####
+#
+# (1) All variables are constrained to be nonnegative
+#
+##### First-Order Optimality Conditions #####
+#
+# Producers_Profit: report of producer profit given the optimal PInput and POutput
+#
+# m_pro_pos: multiplier on Positive production functions
+# m_pro_leo: multiplier on Leontief production functions
+# m_pro_in_bal: multiplier on input commodity balance constraints
+# m_pro_out_bal: multiplier on output commodity balance constraints
+# m_pro_inter_bal: multiplier on intermediate commodity balance constraints
+#
+#####
+
+var Producers_Profit {p in Producers, r in Producers_Regions[p]} := 1.0;
+
+var m_pro_pos {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Positive[p,r,f]} := 1.0;
+var m_pro_leo {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Leontief[p,r,f], (ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} := 1.0;
+
+var m_pro_in_bal {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Inputs[p,r]} := 1.0;
+var m_pro_out_bal {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Outputs[p,r]} := 1.0;
+var m_pro_inter_bal {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Intermeds[p,r]} := 1.0;
+
+subject to
+
+Producers_Profits_Def {p in Producers, r in Producers_Regions[p]}:
+ Producers_Profit[p,r] complements Producers_Profit[p,r] =
+ sum {(ro,ko) in Producers_Outputs[p,r]} ((Producers_Revenues[p,r,ro,ko] -
+ Producers_RRPriceTax_Revenues[p,r,ro,ko]*Producers_RRPriceRate_Revenues[p,r,ro,ko] -
+ Producers_ROPriceTax_Revenues[p,r,ro,ko]*Producers_ROPriceRate_Revenues[p,r,ro,ko]
+ )*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) -
+ Producers_RRQuantityTax_Revenues[p,r,ro,ko]*Producers_RRQuantityRate_Revenues[p,r,ro,ko] -
+ Producers_ROQuantityTax_Revenues[p,r,ro,ko]*Producers_ROQuantityRate_Revenues[p,r,ro,ko]
+ )*Producers_Output[p,r,ro,ko] -
+ sum {(ri,ki) in Producers_Inputs[p,r]} ((Producers_Expenditures[p,r,ri,ki] +
+ Producers_RRPriceTax_Expenditures[p,r,ri,ki]*Producers_RRPriceRate_Expenditures[p,r,ri,ki] +
+ Producers_RIPriceTax_Expenditures[p,r,ri,ki]*Producers_RIPriceRate_Expenditures[p,r,ri,ki]
+ )*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) +
+ Producers_RRQuantityTax_Expenditures[p,r,ri,ki]*Producers_RRQuantityRate_Expenditures[p,r,ri,ki] +
+ Producers_RIQuantityTax_Expenditures[p,r,ri,ki]*Producers_RIQuantityRate_Expenditures[p,r,ri,ki]
+ )*Producers_Input[p,r,ri,ki];
+
+Producers_Tax_Def {rr in Regions}:
+ Producers_Tax[rr] complements Producers_Tax[rr] =
+ sum {p in Producers, r in Producers_Regions[p], (ro,ko) in Producers_Outputs[p,r]: r == rr} (Producers_RRPriceTax_Revenues[p,r,ro,ko]*Producers_RRPriceRate_Revenues[p,r,ro,ko]*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) + Producers_RRQuantityTax_Revenues[p,r,ro,ko]*Producers_RRQuantityRate_Revenues[p,r,ro,ko])*Producers_Output[p,r,ro,ko] +
+ sum {p in Producers, r in Producers_Regions[p], (ro,ko) in Producers_Outputs[p,r]: ro == rr} (Producers_ROPriceTax_Revenues[p,r,ro,ko]*Producers_ROPriceRate_Revenues[p,r,ro,ko]*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) + Producers_ROQuantityTax_Revenues[p,r,ro,ko]*Producers_ROQuantityRate_Revenues[p,r,ro,ko])*Producers_Output[p,r,ro,ko] +
+ sum {p in Producers, r in Producers_Regions[p], (ri,ki) in Producers_Inputs[p,r]: r == rr} (Producers_RRPriceTax_Expenditures[p,r,ri,ki]*Producers_RRPriceRate_Expenditures[p,r,ri,ki]*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) + Producers_RRQuantityTax_Expenditures[p,r,ri,ki]*Producers_RRQuantityRate_Expenditures[p,r,ri,ki])*Producers_Input[p,r,ri,ki] +
+ sum {p in Producers, r in Producers_Regions[p], (ri,ki) in Producers_Inputs[p,r]: ri == rr} (Producers_RIPriceTax_Expenditures[p,r,ri,ki]*Producers_RIPriceRate_Expenditures[p,r,ri,ki]*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) + Producers_RIQuantityTax_Expenditures[p,r,ri,ki]*Producers_RIQuantityRate_Expenditures[p,r,ri,ki])*Producers_Input[p,r,ri,ki];
+
+Producers_Opt_Input {p in Producers, r in Producers_Regions[p], (ri,ki) in Producers_Inputs[p,r]}:
+ 0 <= Producers_Input[p,r,ri,ki] complements ((Producers_Expenditures[p,r,ri,ki] +
+ Producers_RRPriceTax_Expenditures[p,r,ri,ki]*Producers_RRPriceRate_Expenditures[p,r,ri,ki] +
+ Producers_RIPriceTax_Expenditures[p,r,ri,ki]*Producers_RIPriceRate_Expenditures[p,r,ri,ki]
+ )*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) +
+ Producers_RRQuantityTax_Expenditures[p,r,ri,ki]*Producers_RRQuantityRate_Expenditures[p,r,ri,ki] +
+ Producers_RIQuantityTax_Expenditures[p,r,ri,ki]*Producers_RIQuantityRate_Expenditures[p,r,ri,ki]
+ )/Producers_Scale[p,r] - m_pro_in_bal[p,r,ri,ki] >= 0;
+
+Producers_Opt_Output {p in Producers, r in Producers_Regions[p], (ro,ko) in Producers_Outputs[p,r]}:
+ 0 <= Producers_Output[p,r,ro,ko] complements m_pro_out_bal[p,r,ro,ko] - ((Producers_Revenues[p,r,ro,ko] -
+ Producers_RRPriceTax_Revenues[p,r,ro,ko]*Producers_RRPriceRate_Revenues[p,r,ro,ko] -
+ Producers_ROPriceTax_Revenues[p,r,ro,ko]*Producers_ROPriceRate_Revenues[p,r,ro,ko]
+ )*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) -
+ Producers_RRQuantityTax_Revenues[p,r,ro,ko]*Producers_RRQuantityRate_Revenues[p,r,ro,ko] -
+ Producers_ROQuantityTax_Revenues[p,r,ro,ko]*Producers_ROQuantityRate_Revenues[p,r,ro,ko]
+ )/Producers_Scale[p,r] >= 0;
+
+Producers_Opt_Functions_Input {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ri,ki) in Producers_Functions_Inputs[p,r,f]}:
+ 0 <= Producers_Functions_Input[p,r,f,ri,ki] complements (if (ri,ki) in Producers_Inputs[p,r] then m_pro_in_bal[p,r,ri,ki]) + (if (ri,ki) in Producers_Intermeds[p,r] then m_pro_inter_bal[p,r,ri,ki]) - sum {(ro,ko) in Producers_Functions_Positive[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} d_cesp(Producers_Factors_Index[p,r,ri,ki], Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko], Producers_Functions_Sigma[p,r,f,ro,ko], {(r1,k1) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,r1,k1] > 0} (Producers_Factors_Index[p,r,r1,k1], Producers_Functions_Share[p,r,f,ro,ko,r1,k1], Producers_Functions_Input[p,r,f,r1,k1]))*m_pro_pos[p,r,f,ro,ko] - sum {(ro,ko) in Producers_Functions_Leontief[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko]*m_pro_leo[p,r,f,ro,ko,ri,ki] >= 0;
+
+Producers_Opt_Functions_Output {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Outputs[p,r,f]}:
+ 0 <= Producers_Functions_Output[p,r,f,ro,ko] complements (if (ro,ko) in Producers_Functions_Positive[p,r,f] then m_pro_pos[p,r,f,ro,ko]) + (if (ro,ko) in Producers_Functions_Leontief[p,r,f] then sum {(ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} m_pro_leo[p,r,f,ro,ko,ri,ki]) - (if (ro,ko) in Producers_Outputs[p,r] then m_pro_out_bal[p,r,ro,ko]) - (if (ro,ko) in Producers_Intermeds[p,r] then m_pro_inter_bal[p,r,ro,ko]) >= 0;
+
+Producers_Functions_Positive_Def {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Positive[p,r,f]}:
+ 0 <= m_pro_pos[p,r,f,ro,ko] complements cesp(Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko], Producers_Functions_Sigma[p,r,f,ro,ko], {(ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} (Producers_Factors_Index[p,r,ri,ki], Producers_Functions_Share[p,r,f,ro,ko,ri,ki], Producers_Functions_Input[p,r,f,ri,ki])) - Producers_Functions_Output[p,r,f,ro,ko] >= 0;
+
+Producers_Functions_Leontief_Def {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Leontief[p,r,f], (ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0}:
+ 0 <= m_pro_leo[p,r,f,ro,ko,ri,ki] complements Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko]*Producers_Functions_Input[p,r,f,ri,ki] - Producers_Functions_Output[p,r,f,ro,ko] >= 0;
+
+Producers_Inputs_Balance_Def {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Inputs[p,r]}:
+ 0 <= m_pro_in_bal[p,r,rf,kf] complements Producers_Input[p,r,rf,kf] - sum {f in Producers_Functions_Regions[p,r]: (rf,kf) in Producers_Functions_Inputs[p,r,f]} Producers_Functions_Input[p,r,f,rf,kf] >= 0;
+
+Producers_Outputs_Balance_Def {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Outputs[p,r]}:
+ 0 <= m_pro_out_bal[p,r,rf,kf] complements sum {f in Producers_Functions_Regions[p,r]: (rf,kf) in Producers_Functions_Outputs[p,r,f]} Producers_Functions_Output[p,r,f,rf,kf] - Producers_Output[p,r,rf,kf] >= 0;
+
+Producers_Intermeds_Balance_Def {p in Producers, r in Producers_Regions[p], (rf,kf) in Producers_Intermeds[p,r]}:
+ 0 <= m_pro_inter_bal[p,r,rf,kf] complements sum {f in Producers_Functions_Regions[p,r]: (rf,kf) in Producers_Functions_Outputs[p,r,f]} Producers_Functions_Output[p,r,f,rf,kf] - sum {f in Producers_Functions_Regions[p,r]: (rf,kf) in Producers_Functions_Inputs[p,r,f]} Producers_Functions_Input[p,r,f,rf,kf] >= 0;
+
Property changes on: SwiftApps/SEE/trunk/subproblems/producer_graph.mod
___________________________________________________________________
Name: svn:executable
+ *
Added: SwiftApps/SEE/trunk/subproblems/producer_tree.mod
===================================================================
--- SwiftApps/SEE/trunk/subproblems/producer_tree.mod (rev 0)
+++ SwiftApps/SEE/trunk/subproblems/producer_tree.mod 2009-07-07 21:41:29 UTC (rev 3003)
@@ -0,0 +1,97 @@
+##### Producer Variables #####
+#
+# Producers_Input: quantity of the input and intermediate commodities used
+# Producers_Output: quantity of the output commodities produced
+#
+#####
+
+var Producers_Input {p in Producers, r in Producers_Regions[p], Producers_Inputs[p,r] union Producers_Intermeds[p,r]} := 1.0;
+var Producers_Output {p in Producers, r in Producers_Regions[p], Producers_Outputs[p,r]} := 1.0;
+
+check {p in Producers, r in Producers_Regions[p]}: card(Producers_Inputs[p,r] inter Producers_Intermeds[p,r]) = 0;
+check {p in Producers, r in Producers_Regions[p]}: card(Producers_Outputs[p,r] inter Producers_Intermeds[p,r]) = 0;
+
+param Producers_PTR {p in Producers, r in Producers_Regions[p], (ro,ko) in Producers_Outputs[p,r]} :=
+ Producers_RRPriceTax_Revenues[p,r,ro,ko]*Producers_RRPriceRate_Revenues[p,r,ro,ko] +
+ Producers_ROPriceTax_Revenues[p,r,ro,ko]*Producers_ROPriceRate_Revenues[p,r,ro,ko];
+param Producers_QTR {p in Producers, r in Producers_Regions[p], (ro,ko) in Producers_Outputs[p,r]} :=
+ Producers_RRQuantityTax_Revenues[p,r,ro,ko]*Producers_RRQuantityRate_Revenues[p,r,ro,ko] +
+ Producers_ROQuantityTax_Revenues[p,r,ro,ko]*Producers_ROQuantityRate_Revenues[p,r,ro,ko];
+
+param Producers_PTE {p in Producers, r in Producers_Regions[p], (ri,ki) in Producers_Inputs[p,r]} :=
+ Producers_RRPriceTax_Expenditures[p,r,ri,ki]*Producers_RRPriceRate_Expenditures[p,r,ri,ki] +
+ Producers_RIPriceTax_Expenditures[p,r,ri,ki]*Producers_RIPriceRate_Expenditures[p,r,ri,ki];
+param Producers_QTE {p in Producers, r in Producers_Regions[p], (ri,ki) in Producers_Inputs[p,r]} :=
+ Producers_RRQuantityTax_Expenditures[p,r,ri,ki]*Producers_RRQuantityRate_Expenditures[p,r,ri,ki] +
+ Producers_RIQuantityTax_Expenditures[p,r,ri,ki]*Producers_RIQuantityRate_Expenditures[p,r,ri,ki];
+
+Producers_Tax_Def {rr in Regions}:
+ Producers_Tax[rr] complements Producers_Tax[rr] =
+ sum {p in Producers, r in Producers_Regions[p], (ro,ko) in Producers_Outputs[p,r]: r == rr} (Producers_RRPriceTax_Revenues[p,r,ro,ko]*Producers_RRPriceRate_Revenues[p,r,ro,ko]*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) + Producers_RRQuantityTax_Revenues[p,r,ro,ko]*Producers_RRQuantityRate_Revenues[p,r,ro,ko])*Producers_Output[p,r,ro,ko] +
+ sum {p in Producers, r in Producers_Regions[p], (ro,ko) in Producers_Outputs[p,r]: ro == rr} (Producers_ROPriceTax_Revenues[p,r,ro,ko]*Producers_ROPriceRate_Revenues[p,r,ro,ko]*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) + Producers_ROQuantityTax_Revenues[p,r,ro,ko]*Producers_ROQuantityRate_Revenues[p,r,ro,ko])*Producers_Output[p,r,ro,ko] +
+ sum {p in Producers, r in Producers_Regions[p], (ri,ki) in Producers_Inputs[p,r]: r == rr} (Producers_RRPriceTax_Expenditures[p,r,ri,ki]*Producers_RRPriceRate_Expenditures[p,r,ri,ki]*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) + Producers_RRQuantityTax_Expenditures[p,r,ri,ki]*Producers_RRQuantityRate_Expenditures[p,r,ri,ki])*Producers_Input[p,r,ri,ki] +
+ sum {p in Producers, r in Producers_Regions[p], (ri,ki) in Producers_Inputs[p,r]: ri == rr} (Producers_RIPriceTax_Expenditures[p,r,ri,ki]*Producers_RIPriceRate_Expenditures[p,r,ri,ki]*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) + Producers_RIQuantityTax_Expenditures[p,r,ri,ki]*Producers_RIQuantityRate_Expenditures[p,r,ri,ki])*Producers_Input[p,r,ri,ki];
+
+##### Producer Optimization Problem #####
+#
+# maximize Producers_Profits {p in Producers, r in Producers_Regions[p]}:
+# (sum {(ro,ko) in Producers_Outputs[p,r]}
+# ((Producers_Revenues[p,r,ro,ko] - Producers_PTR[p,r,ro,ko])*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) - Producers_QTR[p,r,ro,ko])*Producers_Output[p,r,ro,ko] -
+# sum {(ri,ki) in Producers_Inputs[p,r]}
+# ((Producers_Expenditures[p,r,ri,ki] + Producers_PTE[p,r,ri,ki])*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) + Producers_QTE[p,r,ri,ki])*Producers_Input[p,r,ri,ki]
+# )/Producers_Scale[p,r];
+#
+# Producers_Functions_Positive_Def {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Positive[p,r,f]}:
+# (if (ro,ko) in Producers_Outputs[p,r] then Producers_Output[p,r,ro,ko] else Producers_Input[p,r,ro,ko]) <= cesp(Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko], Producers_Functions_Sigma[p,r,f,ro,ko], {(ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} (Producers_Factors_Index[p,r,ri,ki], Producers_Functions_Share[p,r,f,ro,ko,ri,ki], Producers_Input[p,r,ri,ki]));
+#
+# Producers_Functions_Leontief_Def {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Leontief[p,r,f], (ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0}:
+# (if (ro,ko) in Producers_Outputs[p,r] then Producers_Output[p,r,ro,ko] else Producers_Input[p,r,ro,ko]) <= Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko]*Producers_Input[p,r,ri,ki];
+#
+##### Notes #####
+#
+# (1) All variables are constrained to be nonnegative
+#
+##### First-Order Optimality Conditions #####
+#
+# Producers_Profit: report of producer profit given the optimal PInput and POutput
+#
+# m_pro_pos: multiplier on Positive production functions
+# m_pro_leo: multiplier on Leontief production functions
+#
+#####
+
+var Producers_Profit {p in Producers, r in Producers_Regions[p]} := 1.0;
+
+var m_pro_pos {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Positive[p,r,f]} := 1.0;
+var m_pro_leo {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Leontief[p,r,f], (ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} := 1.0;
+
+subject to
+
+Producers_Profits_Def {p in Producers, r in Producers_Regions[p]}:
+ Producers_Profit[p,r] complements Producers_Profit[p,r] =
+ sum {(ro,ko) in Producers_Outputs[p,r]}
+ ((Producers_Revenues[p,r,ro,ko] - Producers_PTR[p,r,ro,ko])*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) - Producers_QTR[p,r,ro,ko])*Producers_Output[p,r,ro,ko] -
+ sum {(ri,ki) in Producers_Inputs[p,r]}
+ ((Producers_Expenditures[p,r,ri,ki] + Producers_PTE[p,r,ri,ki])*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) + Producers_QTE[p,r,ri,ki])*Producers_Input[p,r,ri,ki];
+
+Producers_Opt_Input {p in Producers, r in Producers_Regions[p], (ri,ki) in Producers_Inputs[p,r] union Producers_Intermeds[p,r]}:
+ 0 <= Producers_Input[p,r,ri,ki] complements
+ (if (ri,ki) in Producers_Inputs[p,r] then ((Producers_Expenditures[p,r,ri,ki] + Producers_PTE[p,r,ri,ki])*(if ki in Homogenous_Commodities then HPrice[ki] else Price[ri,ki]) + Producers_QTE[p,r,ri,ki])/Producers_Scale[p,r]) +
+ (if (ri,ki) in Producers_Intermeds[p,r] then
+ (sum {f in Producers_Functions_Regions[p,r]: (ri,ki) in Producers_Functions_Positive[p,r,f]} m_pro_pos[p,r,f,ri,ki] +
+ sum {f in Producers_Functions_Regions[p,r]: (ri,ki) in Producers_Functions_Leontief[p,r,f]} (sum {(ri2,ki2) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ri,ki,ri2,ki2] > 0} m_pro_leo[p,r,f,ri,ki,ri2,ki2]))) -
+ sum {f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Positive[p,r,f]: (ri,ki) in Producers_Functions_Inputs[p,r,f] and Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} d_cesp(Producers_Factors_Index[p,r,ri,ki], Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko], Producers_Functions_Sigma[p,r,f,ro,ko], {(ri2,ki2) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri2,ki2] > 0} (Producers_Factors_Index[p,r,ri2,ki2], Producers_Functions_Share[p,r,f,ro,ko,ri2,ki2], Producers_Input[p,r,ri2,ki2]))*m_pro_pos[p,r,f,ro,ko] -
+ sum {f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Leontief[p,r,f]: (ri,ki) in Producers_Functions_Inputs[p,r,f] and Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko]*m_pro_leo[p,r,f,ro,ko,ri,ki] >= 0;
+
+Producers_Opt_Output {p in Producers, r in Producers_Regions[p], (ro,ko) in Producers_Outputs[p,r]}:
+ 0 <= Producers_Output[p,r,ro,ko] complements
+ sum {f in Producers_Functions_Regions[p,r]: (ro,ko) in Producers_Functions_Positive[p,r,f]} m_pro_pos[p,r,f,ro,ko] +
+ sum {f in Producers_Functions_Regions[p,r]: (ro,ko) in Producers_Functions_Leontief[p,r,f]} (sum {(ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} m_pro_leo[p,r,f,ro,ko,ri,ki]) -
+ ((Producers_Revenues[p,r,ro,ko] - Producers_PTR[p,r,ro,ko])*(if ko in Homogenous_Commodities then HPrice[ko] else Price[ro,ko]) - Producers_QTR[p,r,ro,ko])/Producers_Scale[p,r] >= 0;
+
+Producers_Functions_Positive_Def {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Positive[p,r,f]}:
+ 0 <= m_pro_pos[p,r,f,ro,ko] complements cesp(Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko], Producers_Functions_Sigma[p,r,f,ro,ko], {(ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0} (Producers_Factors_Index[p,r,ri,ki], Producers_Functions_Share[p,r,f,ro,ko,ri,ki], Producers_Input[p,r,ri,ki])) - (if (ro,ko) in Producers_Intermeds[p,r] then Producers_Input[p,r,ro,ko] else Producers_Output[p,r,ro,ko]) >= 0;
+
+Producers_Functions_Leontief_Def {p in Producers, r in Producers_Regions[p], f in Producers_Functions_Regions[p,r], (ro,ko) in Producers_Functions_Leontief[p,r,f], (ri,ki) in Producers_Functions_Inputs[p,r,f]: Producers_Functions_Share[p,r,f,ro,ko,ri,ki] > 0}:
+ 0 <= m_pro_leo[p,r,f,ro,ko,ri,ki] complements Producers_Functions_Scale[p,r,f,ro,ko]*Producers_Functions_Scale_Factor[p,r,f,ro,ko]*Producers_Input[p,r,ri,ki] - (if (ro,ko) in Producers_Intermeds[p,r] then Producers_Input[p,r,ro,ko] else Producers_Output[p,r,ro,ko]) >= 0;
+
Property changes on: SwiftApps/SEE/trunk/subproblems/producer_tree.mod
___________________________________________________________________
Name: svn:executable
+ *
From noreply at svn.ci.uchicago.edu Wed Jul 8 09:49:40 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 8 Jul 2009 09:49:40 -0500 (CDT)
Subject: [Swift-commit] r3004 - SwiftApps/SEE/trunk
Message-ID: <20090708144941.1518E9CCA8@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-08 09:49:38 -0500 (Wed, 08 Jul 2009)
New Revision: 3004
Modified:
SwiftApps/SEE/trunk/ampl.swift
SwiftApps/SEE/trunk/ff-grid.xml
SwiftApps/SEE/trunk/runampl.sh
SwiftApps/SEE/trunk/swift.properties
SwiftApps/SEE/trunk/template
Log:
update for 5k run
Modified: SwiftApps/SEE/trunk/ampl.swift
===================================================================
--- SwiftApps/SEE/trunk/ampl.swift 2009-07-07 21:41:29 UTC (rev 3003)
+++ SwiftApps/SEE/trunk/ampl.swift 2009-07-08 14:49:38 UTC (rev 3004)
@@ -8,7 +8,7 @@
run_ampl instanceID @filename(temp) @filename(mod) @filename(process) @filename(output) @filename(so) @filename(tree) stdout=@filename(ofile);
}
-int runs[]=[1:2000];
+int runs[]=[5001:10000];
foreach i in runs {
Template temp <"template">;
AMPLIn mod <"armington.mod">;
Modified: SwiftApps/SEE/trunk/ff-grid.xml
===================================================================
--- SwiftApps/SEE/trunk/ff-grid.xml 2009-07-07 21:41:29 UTC (rev 3003)
+++ SwiftApps/SEE/trunk/ff-grid.xml 2009-07-08 14:49:38 UTC (rev 3004)
@@ -3,17 +3,13 @@
-
-
- 10
- 18.0
+
+ 10000.0
+ 10.0
/panfs/panasas/CMS/data/SEE/workdir
-
Modified: SwiftApps/SEE/trunk/runampl.sh
===================================================================
--- SwiftApps/SEE/trunk/runampl.sh 2009-07-07 21:41:29 UTC (rev 3003)
+++ SwiftApps/SEE/trunk/runampl.sh 2009-07-08 14:49:38 UTC (rev 3004)
@@ -8,5 +8,5 @@
#-tc.file ./tc.data ampl.swift
#swift -runid teraport -sites.file ./sites-teraport.xml \
#-tc.file ./tc.data ampl.swift
-#swift -runid firefly_run -config swift.properties \
- #-sites.file ./ff-grid.xml -tc.file ./tc.data ampl.swift
+swift -runid run3k_firefly -config swift.properties \
+ -sites.file ./ff-grid.xml -tc.file ./tc.data ampl.swift
Modified: SwiftApps/SEE/trunk/swift.properties
===================================================================
--- SwiftApps/SEE/trunk/swift.properties 2009-07-07 21:41:29 UTC (rev 3003)
+++ SwiftApps/SEE/trunk/swift.properties 2009-07-08 14:49:38 UTC (rev 3004)
@@ -149,7 +149,7 @@
# Default: false
#
-wrapperlog.always.transfer=false
+wrapperlog.always.transfer=true
###########################################################################
# Throttling options #
Modified: SwiftApps/SEE/trunk/template
===================================================================
--- SwiftApps/SEE/trunk/template 2009-07-07 21:41:29 UTC (rev 3003)
+++ SwiftApps/SEE/trunk/template 2009-07-08 14:49:38 UTC (rev 3004)
@@ -1,4 +1,4 @@
-option ampl_include "/home/zzhang/SEE/static/instances/test/@INSTANCE_ID@\
+option ampl_include "/panfs/panasas/CMS/data/SEE/run5kdata/@INSTANCE_ID@\
.";
option solver pathampl;
option path_options "nms_ini_ref_min=1e8 cra_met=none cra_per=no fac_upd_lim=200";
From noreply at svn.ci.uchicago.edu Mon Jul 13 02:26:20 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 13 Jul 2009 02:26:20 -0500 (CDT)
Subject: [Swift-commit] r3005 - trunk/tests/language-behaviour
Message-ID: <20090713072620.0B1079CCA9@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-13 02:26:19 -0500 (Mon, 13 Jul 2009)
New Revision: 3005
Added:
trunk/tests/language-behaviour/07554-ext-mapper-struct.a.out
trunk/tests/language-behaviour/07554-ext-mapper-struct.b.out
trunk/tests/language-behaviour/07554-ext-mapper-struct.sh
trunk/tests/language-behaviour/07554-ext-mapper-struct.swift
Log:
test for ext mapper with structs
Added: trunk/tests/language-behaviour/07554-ext-mapper-struct.a.out
===================================================================
--- trunk/tests/language-behaviour/07554-ext-mapper-struct.a.out (rev 0)
+++ trunk/tests/language-behaviour/07554-ext-mapper-struct.a.out 2009-07-13 07:26:19 UTC (rev 3005)
@@ -0,0 +1 @@
+1st
Added: trunk/tests/language-behaviour/07554-ext-mapper-struct.b.out
===================================================================
--- trunk/tests/language-behaviour/07554-ext-mapper-struct.b.out (rev 0)
+++ trunk/tests/language-behaviour/07554-ext-mapper-struct.b.out 2009-07-13 07:26:19 UTC (rev 3005)
@@ -0,0 +1 @@
+2nd
Added: trunk/tests/language-behaviour/07554-ext-mapper-struct.sh
===================================================================
--- trunk/tests/language-behaviour/07554-ext-mapper-struct.sh (rev 0)
+++ trunk/tests/language-behaviour/07554-ext-mapper-struct.sh 2009-07-13 07:26:19 UTC (rev 3005)
@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "eerste 07554-ext-mapper-struct.a.out"
+echo "twede 07554-ext-mapper-struct.b.out"
Property changes on: trunk/tests/language-behaviour/07554-ext-mapper-struct.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/language-behaviour/07554-ext-mapper-struct.swift
===================================================================
--- trunk/tests/language-behaviour/07554-ext-mapper-struct.swift (rev 0)
+++ trunk/tests/language-behaviour/07554-ext-mapper-struct.swift 2009-07-13 07:26:19 UTC (rev 3005)
@@ -0,0 +1,18 @@
+type messagefile;
+
+type struct {
+messagefile eerste;
+messagefile twede;
+};
+
+(messagefile t) write(string s) {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+struct outfiles ;
+
+outfiles.eerste = write("1st");
+outfiles.twede = write("2nd");
+
From noreply at svn.ci.uchicago.edu Tue Jul 14 02:59:08 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Jul 2009 02:59:08 -0500 (CDT)
Subject: [Swift-commit] r3006 - nmi-build-test/submit-machine
Message-ID: <20090714075908.304F39CCC4@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-14 02:59:07 -0500 (Tue, 14 Jul 2009)
New Revision: 3006
Modified:
nmi-build-test/submit-machine/kickstart-hourly
Log:
Modified: nmi-build-test/submit-machine/kickstart-hourly
===================================================================
--- nmi-build-test/submit-machine/kickstart-hourly 2009-07-13 07:26:19 UTC (rev 3005)
+++ nmi-build-test/submit-machine/kickstart-hourly 2009-07-14 07:59:07 UTC (rev 3006)
@@ -2,6 +2,7 @@
component = kickstart
component_version = HEAD
+
run_type = build
inputs = kickstart.svn, swiftnmi.svn
From noreply at svn.ci.uchicago.edu Tue Jul 14 17:06:08 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Jul 2009 17:06:08 -0500 (CDT)
Subject: [Swift-commit] r3007 - SwiftApps/SEE/trunk
Message-ID: <20090714220608.9E9589CCC4@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-14 17:06:08 -0500 (Tue, 14 Jul 2009)
New Revision: 3007
Added:
SwiftApps/SEE/trunk/cmd_mapper.sh
SwiftApps/SEE/trunk/instance_mapper.sh
Modified:
SwiftApps/SEE/trunk/ampl.swift
SwiftApps/SEE/trunk/run_ampl
SwiftApps/SEE/trunk/runampl.sh
Log:
Struct-urized swift script. Added required mappers for clean code
Modified: SwiftApps/SEE/trunk/ampl.swift
===================================================================
--- SwiftApps/SEE/trunk/ampl.swift 2009-07-14 07:59:07 UTC (rev 3006)
+++ SwiftApps/SEE/trunk/ampl.swift 2009-07-14 22:06:08 UTC (rev 3007)
@@ -1,29 +1,57 @@
-type Template {}
-type AMPLIn{}
-type AMPLOut {}
-type STDOut {}
+type Template;
+type AmplIn;
+type StdOut;
-app (AMPLOut expend, AMPLOut limits, AMPLOut price, AMPLOut ratio, AMPLOut solve, STDOut ofile) run_ampl (string instanceID, Template temp, AMPLIn mod, AMPLIn process, AMPLIn output, AMPLIn so, AMPLIn tree)
+type AmplCmd {
+ Template temp;
+ AmplIn mod;
+ AmplIn process;
+ AmplIn output;
+ AmplIn so;
+ AmplIn tree;
+}
+
+type ExpendDat;
+type LimitsDat;
+type PriceDat;
+type RatioDat;
+type SolveDat;
+
+type ExpendOut;
+type PriceOut;
+type RatioOut;
+
+type AmplFilter {
+ ExpendOut expend_out;
+ PriceOut price_out;
+ RatioOut ratio_out;
+}
+
+type AmplResult {
+ ExpendDat expend;
+ LimitsDat limits;
+ PriceDat price;
+ RatioDat ratio;
+ SolveDat solve;
+ StdOut ofile;
+ AmplFilter out;
+}
+
+app (AmplResult result) run_ampl (string instanceID, AmplCmd cmd)
{
- run_ampl instanceID @filename(temp) @filename(mod) @filename(process) @filename(output) @filename(so) @filename(tree) stdout=@filename(ofile);
+ run_ampl instanceID @filename(cmd.temp)
+ @filename(cmd.mod) @filename(cmd.process)
+ @filename(cmd.output) @filename(cmd.so) @filename(cmd.tree)
+ stdout=@filename(result.ofile);
}
-int runs[]=[5001:10000];
+AmplCmd const_cmd ;
+
+int runs[]=[2001:2002];
foreach i in runs {
- Template temp <"template">;
- AMPLIn mod <"armington.mod">;
- AMPLIn process <"armington_process.cmd">;
- AMPLIn output <"armington_output.cmd">;
- AMPLIn so <"ces.so">;
- AMPLIn tree <"subproblems/producer_tree.mod">;
string instanceID = @strcat("run", i);
- AMPLOut expend ;
- AMPLOut limits ;
- AMPLOut price ;
- AMPLOut ratio ;
- AMPLOut solve ;
- STDOut ofile ;
- (expend, limits, price, ratio, solve, ofile)=run_ampl(instanceID, temp, mod, process, output, tree, so);
+ AmplResult res ;
+ res = run_ampl(instanceID, const_cmd);
}
Added: SwiftApps/SEE/trunk/cmd_mapper.sh
===================================================================
--- SwiftApps/SEE/trunk/cmd_mapper.sh (rev 0)
+++ SwiftApps/SEE/trunk/cmd_mapper.sh 2009-07-14 22:06:08 UTC (rev 3007)
@@ -0,0 +1,8 @@
+#!/bin/bash
+echo "temp template"
+echo "mod armington.mod"
+echo "process armington_process.cmd"
+echo "output armington_output.cmd"
+echo "so ces.so"
+echo "tree subproblems/producer_tree.mod"
+
Property changes on: SwiftApps/SEE/trunk/cmd_mapper.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: SwiftApps/SEE/trunk/instance_mapper.sh
===================================================================
--- SwiftApps/SEE/trunk/instance_mapper.sh (rev 0)
+++ SwiftApps/SEE/trunk/instance_mapper.sh 2009-07-14 22:06:08 UTC (rev 3007)
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+while getopts ":i:" options; do
+ case $options in
+ i) export instance=$OPTARG ;;
+ *) exit 1;;
+ esac
+done
+
+echo "expend result/$instance/expend.dat";
+echo "limits result/$instance/limits.dat";
+echo "price result/$instance/price.dat";
+echo "ratio result/$instance/ratio.dat";
+echo "solve result/$instance/solve.dat";
+
+echo "ofile result/$instance/stdout";
+
+echo "out.expend_out result/$instance/expend.out";
+echo "out.price_out result/$instance/price.out";
+echo "out.ratio_out result/$instance/ratio.out";
Property changes on: SwiftApps/SEE/trunk/instance_mapper.sh
___________________________________________________________________
Name: svn:executable
+ *
Modified: SwiftApps/SEE/trunk/run_ampl
===================================================================
--- SwiftApps/SEE/trunk/run_ampl 2009-07-14 07:59:07 UTC (rev 3006)
+++ SwiftApps/SEE/trunk/run_ampl 2009-07-14 22:06:08 UTC (rev 3007)
@@ -1,5 +1,5 @@
#!/bin/bash
-export PATH=$PATH:/home/zzhang/amplbin/
+export PATH=$PATH:/panfs/panasas/CMS/data/amplbin/
export PATH_LICENSE_STRING="2640088036&Todd_Munson&Argonne_National_Laboratory&&USR&&14_12_2001&1000&COMP&GEN&0_0_0&0_0_0&0&0_0"
if [ $# -lt 1 ]; then
Modified: SwiftApps/SEE/trunk/runampl.sh
===================================================================
--- SwiftApps/SEE/trunk/runampl.sh 2009-07-14 07:59:07 UTC (rev 3006)
+++ SwiftApps/SEE/trunk/runampl.sh 2009-07-14 22:06:08 UTC (rev 3007)
@@ -8,5 +8,5 @@
#-tc.file ./tc.data ampl.swift
#swift -runid teraport -sites.file ./sites-teraport.xml \
#-tc.file ./tc.data ampl.swift
-swift -runid run3k_firefly -config swift.properties \
+swift -runid testing -config swift.properties \
-sites.file ./ff-grid.xml -tc.file ./tc.data ampl.swift
From noreply at svn.ci.uchicago.edu Tue Jul 14 21:04:26 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Jul 2009 21:04:26 -0500 (CDT)
Subject: [Swift-commit] r3008 - SwiftApps/SEE/trunk
Message-ID: <20090715020426.227839CCC4@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-14 21:04:25 -0500 (Tue, 14 Jul 2009)
New Revision: 3008
Modified:
SwiftApps/SEE/trunk/instance_mapper.sh
Log:
swift code now compiles with struct hack from ben
Modified: SwiftApps/SEE/trunk/instance_mapper.sh
===================================================================
--- SwiftApps/SEE/trunk/instance_mapper.sh 2009-07-14 22:06:08 UTC (rev 3007)
+++ SwiftApps/SEE/trunk/instance_mapper.sh 2009-07-15 02:04:25 UTC (rev 3008)
@@ -15,6 +15,7 @@
echo "ofile result/$instance/stdout";
+echo "out null";
echo "out.expend_out result/$instance/expend.out";
echo "out.price_out result/$instance/price.out";
echo "out.ratio_out result/$instance/ratio.out";
From noreply at svn.ci.uchicago.edu Tue Jul 14 21:45:16 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 14 Jul 2009 21:45:16 -0500 (CDT)
Subject: [Swift-commit] r3009 - SwiftApps/SIDGrid/scripts
Message-ID: <20090715024516.AE3759CCA9@vm-125-59.ci.uchicago.edu>
Author: skenny
Date: 2009-07-14 21:45:16 -0500 (Tue, 14 Jul 2009)
New Revision: 3009
Modified:
SwiftApps/SIDGrid/scripts/Mediator.py
Log:
single vox flag
Modified: SwiftApps/SIDGrid/scripts/Mediator.py
===================================================================
--- SwiftApps/SIDGrid/scripts/Mediator.py 2009-07-15 02:04:25 UTC (rev 3008)
+++ SwiftApps/SIDGrid/scripts/Mediator.py 2009-07-15 02:45:16 UTC (rev 3009)
@@ -20,15 +20,17 @@
self.r_swift_args = ""
self.r_script = ""
self.r_infile = ""
- self.begin_vox = ""
- self.end_vox = ""
+ self.begin_vox = "1"
+ self.end_vox = "2"
+ self.vox = -1
self.subject = ""
- self.batchstep = ""
+ self.batchstep = "1"
self.helpmsg = ""
self.outprefix = "defaultoutname"
self.all_queries = []
self.qout_r_args = []
self.g_db = None
+ self.voxfile = "false"
#..........................................................................
@@ -111,7 +113,11 @@
elif o == "--batchstep":
self.batchstep = argstr[i+1]
elif o == "--outprefix":
- self.outprefix = argstr[i+1]
+ self.outprefix = argstr[i+1]
+ elif o == "--vox_file":
+ self.voxfile = argstr[i+1]
+ elif o == "--vox":
+ self.vox = argstr[i+1]
elif (o.find("--")>-1):
print "o is this "+o
print "and i is this "+str(i)
@@ -124,7 +130,7 @@
def gethelp(self,option):
helpmsg = """
Unknown Option: """+option+"""
- usage: --user, --conf, --db, --host, --query, --r_swift_args, --r_script, --r_infile, --outprefix, --begin_vox, --end_vox, --batchstep
+ usage: --user, --conf, --db, --host, --query, --r_swift_args, --r_script, --r_infile, --outprefix, --begin_vox, --end_vox, --batchstep, --vox
"""
return helpmsg
@@ -149,21 +155,42 @@
#.....................................................................
def gen_queries(self):
- bgn = int(self.begin_vox)
- end = int(self.end_vox)
- stp = int(self.batchstep)
- for i in range(bgn,end,stp):
- beginbatch = i
- endbatch = i+stp-1
- tmp = self.query.replace("BEGIN_BATCH",str(beginbatch))
- tmp2= tmp.replace("END_BATCH",str(endbatch))
- tmp3= tmp2.replace("SUBJECT", self.subject)
- self.all_queries.append(tmp3)
- self.qout_r_args.append(str(beginbatch)+"_"+str(endbatch))
+ if(self.vox > -1):
+ tmp = self.query.replace("VOX",str(self.vox))
+ tmp2 = tmp.replace("SUBJECT", self.subject)
+ self.all_queries.append(tmp2)
+ self.qout_r_args.append(str(self.vox))
+ else:
+ bgn = int(self.begin_vox)
+ end = int(self.end_vox)
+ stp = int(self.batchstep)
+ for i in range(bgn,end,stp):
+ beginbatch = i
+ endbatch = i+stp-1
+ tmp = self.query.replace("BEGIN_BATCH",str(beginbatch))
+ tmp2= tmp.replace("END_BATCH",str(endbatch))
+ tmp3= tmp2.replace("SUBJECT", self.subject)
+ self.all_queries.append(tmp3)
+ self.qout_r_args.append(str(beginbatch)+"_"+str(endbatch))
pass
#.......................................................................
+ def gen_queries_noncontig(self):
+ allqueries = []
+ query = self.query
+ qcount = 0
+ voxfname = self.voxfile
+ voxfile = open ( voxfname, "r" )
+ for line in voxfile.readlines ():
+ tmp = query.replace("VOX_BATCH", "("+line.strip()+");")
+ self.all_queries.append(tmp)
+ self.qout_r_args.append(str(qcount)+"_")
+ qcount = qcount + 1
+ voxfile.close()
+
+ #.......................................................................
+
def run_query(self, query):
print "RUNNING QUERY: "+query
self.passwd = self.load('PASSWORD')
@@ -208,7 +235,10 @@
med = Mediator ()
med.get_opts(sys.argv)
-med.gen_queries()
+if med.voxfile == "false":
+ med.gen_queries()
+else:
+ med.gen_queries_noncontig()
n=0
for q in med.all_queries:
med.run_query(q)
From noreply at svn.ci.uchicago.edu Wed Jul 15 09:00:13 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 15 Jul 2009 09:00:13 -0500 (CDT)
Subject: [Swift-commit] r3010 - trunk/tests/language-behaviour
Message-ID: <20090715140013.C76E39CCA9@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-15 09:00:13 -0500 (Wed, 15 Jul 2009)
New Revision: 3010
Removed:
trunk/tests/language-behaviour/07554-ext-mapper-struct.a.out
trunk/tests/language-behaviour/07554-ext-mapper-struct.b.out
Log:
remove accidentally committed temporary files
Deleted: trunk/tests/language-behaviour/07554-ext-mapper-struct.a.out
===================================================================
--- trunk/tests/language-behaviour/07554-ext-mapper-struct.a.out 2009-07-15 02:45:16 UTC (rev 3009)
+++ trunk/tests/language-behaviour/07554-ext-mapper-struct.a.out 2009-07-15 14:00:13 UTC (rev 3010)
@@ -1 +0,0 @@
-1st
Deleted: trunk/tests/language-behaviour/07554-ext-mapper-struct.b.out
===================================================================
--- trunk/tests/language-behaviour/07554-ext-mapper-struct.b.out 2009-07-15 02:45:16 UTC (rev 3009)
+++ trunk/tests/language-behaviour/07554-ext-mapper-struct.b.out 2009-07-15 14:00:13 UTC (rev 3010)
@@ -1 +0,0 @@
-2nd
From noreply at svn.ci.uchicago.edu Wed Jul 15 09:00:48 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 15 Jul 2009 09:00:48 -0500 (CDT)
Subject: [Swift-commit] r3011 - in trunk: src/org/griphyn/vdl/karajan/lib
src/org/griphyn/vdl/mapping tests/language-behaviour
Message-ID: <20090715140048.54CDF9CCA9@vm-125-59.ci.uchicago.edu>
Author: benc
Date: 2009-07-15 09:00:48 -0500 (Wed, 15 Jul 2009)
New Revision: 3011
Added:
trunk/tests/language-behaviour/07555-ext-mapper-twostruct.sh
trunk/tests/language-behaviour/07555-ext-mapper-twostruct.swift
trunk/tests/language-behaviour/075551-ext-mapper-twostruct-singleproducer.swift
Modified:
trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
Log:
Fix incorrect attempt to map paths that don't map to filenames.
Two additional tests:
07555-ext-mapper-twostruct passed before this commit.
075551-ext-mapper-twostruct-singleproducer failed before this commit but
passes now.
Modified: trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java
===================================================================
--- trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2009-07-15 14:00:13 UTC (rev 3010)
+++ trunk/src/org/griphyn/vdl/karajan/lib/VDLFunction.java 2009-07-15 14:00:48 UTC (rev 3011)
@@ -196,8 +196,9 @@
try {
if (var.getType().isArray()) {
return leavesFileNames(var);
- }
- else {
+ } else if(var.getType().getFields().size()>0) {
+ return leavesFileNames(var);
+ } else {
return new String[] { leafFileName(var) };
}
}
Modified: trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java
===================================================================
--- trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-07-15 14:00:13 UTC (rev 3010)
+++ trunk/src/org/griphyn/vdl/mapping/AbstractDataNode.java 2009-07-15 14:00:48 UTC (rev 3011)
@@ -378,7 +378,7 @@
// Path fullPath =
// parentPath.addLast(mapper.getField().getName());
Path fullPath = parentPath.addLast(field.getName());
- if (!mapper.field.getType().isPrimitive() && !mapper.isArray()) {
+ if (!mapper.field.getType().isPrimitive() && !mapper.isArray() && mapper.field.getType().getFields().size() == 0) {
list.add(fullPath);
}
else {
Added: trunk/tests/language-behaviour/07555-ext-mapper-twostruct.sh
===================================================================
--- trunk/tests/language-behaviour/07555-ext-mapper-twostruct.sh (rev 0)
+++ trunk/tests/language-behaviour/07555-ext-mapper-twostruct.sh 2009-07-15 14:00:48 UTC (rev 3011)
@@ -0,0 +1,5 @@
+#!/bin/bash
+echo "eerste 07555-ext-mapper-twostruct.a.out"
+echo "twede 07555-ext-mapper-twostruct.b.out"
+echo "derde.links 07555-ext-mapper-twostruct.cl.out"
+echo "derde.rechts 07555-ext-mapper-twostruct.cr.out"
Property changes on: trunk/tests/language-behaviour/07555-ext-mapper-twostruct.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: trunk/tests/language-behaviour/07555-ext-mapper-twostruct.swift
===================================================================
--- trunk/tests/language-behaviour/07555-ext-mapper-twostruct.swift (rev 0)
+++ trunk/tests/language-behaviour/07555-ext-mapper-twostruct.swift 2009-07-15 14:00:48 UTC (rev 3011)
@@ -0,0 +1,26 @@
+type file;
+
+type structInner {
+ file links;
+ file rechts;
+}
+
+type struct {
+ file eerste;
+ file twede;
+ structInner derde;
+};
+
+(file t) write(string s) {
+ app {
+ echo s stdout=@filename(t);
+ }
+}
+
+struct outfiles ;
+
+outfiles.eerste = write("1st");
+outfiles.twede = write("2nd");
+outfiles.derde.links = write("3l");
+outfiles.derde.rechts = write("3r");
+
Added: trunk/tests/language-behaviour/075551-ext-mapper-twostruct-singleproducer.swift
===================================================================
--- trunk/tests/language-behaviour/075551-ext-mapper-twostruct-singleproducer.swift (rev 0)
+++ trunk/tests/language-behaviour/075551-ext-mapper-twostruct-singleproducer.swift 2009-07-15 14:00:48 UTC (rev 3011)
@@ -0,0 +1,23 @@
+type file;
+
+type structInner {
+ file links;
+ file rechts;
+}
+
+type struct {
+ file eerste;
+ file twede;
+ structInner derde;
+};
+
+(struct t) singlewrite() {
+ app {
+ touch @filenames(t);
+ }
+}
+
+struct outfiles ;
+
+outfiles = singlewrite();
+
From noreply at svn.ci.uchicago.edu Wed Jul 15 11:50:43 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 15 Jul 2009 11:50:43 -0500 (CDT)
Subject: [Swift-commit] r3012 - SwiftApps/SEE/trunk
Message-ID: <20090715165043.8F2169CC98@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-15 11:50:43 -0500 (Wed, 15 Jul 2009)
New Revision: 3012
Modified:
SwiftApps/SEE/trunk/run_ampl
Log:
Incorporated mike's filtering scripts
Modified: SwiftApps/SEE/trunk/run_ampl
===================================================================
--- SwiftApps/SEE/trunk/run_ampl 2009-07-15 14:00:48 UTC (rev 3011)
+++ SwiftApps/SEE/trunk/run_ampl 2009-07-15 16:50:43 UTC (rev 3012)
@@ -11,20 +11,67 @@
{
repl=""
for v in $*; do
-# echo substituting $v = ${!v}
val=${!v}
repl="$repl -e 's,@${v}@,$val,'"
done
-# echo Running "sed $repl <$paramtemplate >$paramfile"
eval sed $repl <$paramtemplate >$paramfile
}
+# Extraction scripts
+
+# Expend
+extract_expend()
+{
+ echo 'YEAR REGION PROD CON1 '
+
+ egrep '\+|^ *ELC|AGF.*OIL' $1 | sed -e 's/AGF.*OIL.*//' | egrep -v 'LND|LAB|CAP|NAT' |
+
+ awk '
+
+ BEGIN { regnum=0; year=2004 }
+
+ NF==1 { region=$1; regnum++; if ( regnum % 16 == 0 ) year++ }
+ NF!=1 { print year, region, $1, $19 }
+
+ '
+}
+
+# Price
+extract_price()
+{
+ grep 'OIL_I' $1 | awk '{ print $2 }'
+}
+
+# Ratio
+extract_ratio()
+{
+ echo -n 'YEAR REGION PRODUCT '
+ head -1 $1 | sed -e 's/^ *USA *//' -e 's/ *USA.*//'
+
+ egrep '^ *OIL|^ *COL|^ *GAS|^ *ELC|^ *PTL|AGF *OIL' $1 | sed -e 's/ .*//' |
+
+ awk '
+
+ BEGIN { regnum=0; year=2004 }
+
+ NF==1 { region=$1; regnum++; if ( regnum % 16 == 0 ) year++ }
+ # NF!=1 { print year, region, $0 }
+ NF!=1 {
+ printf("%s %s ", year, region);
+ for(f=1;f<=19;f++) { printf(" %s",$(f)); }
+ printf("\n");
+ }
+
+ '
+}
+
+# main loop
+
paramtemplate=template
amplapp=ampl
paramfile=arm_test.cmd
INSTANCE_ID=$1
-
subval INSTANCE_ID
$amplapp $paramfile
@@ -40,6 +87,11 @@
mv solve.dat result/${INSTANCE_ID}/solve.dat
exit 0
fi
+
+# Filtering
+extract_expend expend.dat > expend.out
+extract_price price.dat > price.out
+extract_ratio ratio.dat > ratio.out
mkdir -p result/${INSTANCE_ID}
mv expend.dat result/${INSTANCE_ID}/expend.dat
@@ -48,4 +100,8 @@
mv ratio.dat result/${INSTANCE_ID}/ratio.dat
mv solve.dat result/${INSTANCE_ID}/solve.dat
+mv expend.out result/${INSTANCE_ID}/expend.out
+mv price.out result/${INSTANCE_ID}/price.out
+mv ratio.out result/${INSTANCE_ID}/ratio.out
+
exit 0
From noreply at svn.ci.uchicago.edu Wed Jul 15 14:00:47 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 15 Jul 2009 14:00:47 -0500 (CDT)
Subject: [Swift-commit] r3013 - SwiftApps/SEE/trunk
Message-ID: <20090715190047.2907E9CC98@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-15 14:00:46 -0500 (Wed, 15 Jul 2009)
New Revision: 3013
Modified:
SwiftApps/SEE/trunk/run_ampl
Log:
Fixe for off-by-one shifting of RUS
Modified: SwiftApps/SEE/trunk/run_ampl
===================================================================
--- SwiftApps/SEE/trunk/run_ampl 2009-07-15 16:50:43 UTC (rev 3012)
+++ SwiftApps/SEE/trunk/run_ampl 2009-07-15 19:00:46 UTC (rev 3013)
@@ -28,9 +28,9 @@
awk '
- BEGIN { regnum=0; year=2004 }
+ BEGIN { regnum=0; year=2003 }
- NF==1 { region=$1; regnum++; if ( regnum % 16 == 0 ) year++ }
+ NF==1 { region=$1; if ( regnum % 16 == 0 ) year++; regnum++}
NF!=1 { print year, region, $1, $19 }
'
@@ -52,9 +52,9 @@
awk '
- BEGIN { regnum=0; year=2004 }
+ BEGIN { regnum=0; year=2003 }
- NF==1 { region=$1; regnum++; if ( regnum % 16 == 0 ) year++ }
+ NF==1 { region=$1; if ( regnum % 16 == 0 ) year++; regnum++ }
# NF!=1 { print year, region, $0 }
NF!=1 {
printf("%s %s ", year, region);
From noreply at svn.ci.uchicago.edu Wed Jul 15 15:01:43 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Wed, 15 Jul 2009 15:01:43 -0500 (CDT)
Subject: [Swift-commit] r3014 - SwiftApps/SEE/trunk
Message-ID: <20090715200143.31C9B9CCA9@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-15 15:01:42 -0500 (Wed, 15 Jul 2009)
New Revision: 3014
Modified:
SwiftApps/SEE/trunk/instance_mapper.sh
SwiftApps/SEE/trunk/runampl.sh
SwiftApps/SEE/trunk/template
Log:
update for 3k run
Modified: SwiftApps/SEE/trunk/instance_mapper.sh
===================================================================
--- SwiftApps/SEE/trunk/instance_mapper.sh 2009-07-15 19:00:46 UTC (rev 3013)
+++ SwiftApps/SEE/trunk/instance_mapper.sh 2009-07-15 20:01:42 UTC (rev 3014)
@@ -15,7 +15,6 @@
echo "ofile result/$instance/stdout";
-echo "out null";
echo "out.expend_out result/$instance/expend.out";
echo "out.price_out result/$instance/price.out";
echo "out.ratio_out result/$instance/ratio.out";
Modified: SwiftApps/SEE/trunk/runampl.sh
===================================================================
--- SwiftApps/SEE/trunk/runampl.sh 2009-07-15 19:00:46 UTC (rev 3013)
+++ SwiftApps/SEE/trunk/runampl.sh 2009-07-15 20:01:42 UTC (rev 3014)
@@ -8,5 +8,5 @@
#-tc.file ./tc.data ampl.swift
#swift -runid teraport -sites.file ./sites-teraport.xml \
#-tc.file ./tc.data ampl.swift
-swift -runid testing -config swift.properties \
+swift -runid run3k_firefly -config swift.properties \
-sites.file ./ff-grid.xml -tc.file ./tc.data ampl.swift
Modified: SwiftApps/SEE/trunk/template
===================================================================
--- SwiftApps/SEE/trunk/template 2009-07-15 19:00:46 UTC (rev 3013)
+++ SwiftApps/SEE/trunk/template 2009-07-15 20:01:42 UTC (rev 3014)
@@ -1,4 +1,4 @@
-option ampl_include "/panfs/panasas/CMS/data/SEE/run5kdata/@INSTANCE_ID@\
+option ampl_include "/panfs/panasas/CMS/data/SEE/run3kdata/@INSTANCE_ID@\
.";
option solver pathampl;
option path_options "nms_ini_ref_min=1e8 cra_met=none cra_per=no fac_upd_lim=200";
From noreply at svn.ci.uchicago.edu Fri Jul 17 09:33:53 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Fri, 17 Jul 2009 09:33:53 -0500 (CDT)
Subject: [Swift-commit] r3015 - SwiftApps/SIDGrid/scripts
Message-ID: <20090717143353.B78649CC98@vm-125-59.ci.uchicago.edu>
Author: skenny
Date: 2009-07-17 09:33:53 -0500 (Fri, 17 Jul 2009)
New Revision: 3015
Modified:
SwiftApps/SIDGrid/scripts/Mediator.py
Log:
fixed ranger for ts batch
Modified: SwiftApps/SIDGrid/scripts/Mediator.py
===================================================================
--- SwiftApps/SIDGrid/scripts/Mediator.py 2009-07-15 20:01:42 UTC (rev 3014)
+++ SwiftApps/SIDGrid/scripts/Mediator.py 2009-07-17 14:33:53 UTC (rev 3015)
@@ -17,11 +17,14 @@
self.db_name = ""
self.host = ""
self.query = ""
+ self.subquery = ""
self.r_swift_args = ""
self.r_script = ""
self.r_infile = ""
self.begin_vox = "1"
self.end_vox = "2"
+ self.begin_ts = "-1"
+ self.end_ts = "-1"
self.vox = -1
self.subject = ""
self.batchstep = "1"
@@ -114,10 +117,16 @@
self.batchstep = argstr[i+1]
elif o == "--outprefix":
self.outprefix = argstr[i+1]
+ elif o == "--begin_ts":
+ self.begin_ts = argstr[i+1]
+ elif o == "--end_ts":
+ self.end_ts = argstr[i+1]
elif o == "--vox_file":
self.voxfile = argstr[i+1]
elif o == "--vox":
self.vox = argstr[i+1]
+ elif o == "--subquery":
+ self.subquery = argstr[i+1]
elif (o.find("--")>-1):
print "o is this "+o
print "and i is this "+str(i)
@@ -130,7 +139,7 @@
def gethelp(self,option):
helpmsg = """
Unknown Option: """+option+"""
- usage: --user, --conf, --db, --host, --query, --r_swift_args, --r_script, --r_infile, --outprefix, --begin_vox, --end_vox, --batchstep, --vox
+ usage: --db --user --conf --host --query --r_swift_args --rscript --r_infile --begin_vox --end_vox --subject --batchstep --outprefix --vox_file --vox --begin_ts --end_ts --subquery
"""
return helpmsg
@@ -158,8 +167,23 @@
if(self.vox > -1):
tmp = self.query.replace("VOX",str(self.vox))
tmp2 = tmp.replace("SUBJECT", self.subject)
- self.all_queries.append(tmp2)
- self.qout_r_args.append(str(self.vox))
+ if(self.begin_ts < 0):
+ self.all_queries.append(tmp2)
+ self.qout_r_args.append(str(self.vox))
+ else:
+ self.query = tmp
+ if(self.begin_ts > -1):
+ bgn = int(self.begin_ts)
+ end = int(self.end_ts)
+ querystr = ""
+ for h in range(bgn,end):
+ tmp = self.subquery.replace("TSVAR", str(h))
+ querystr = querystr+tmp+", "
+ querystr = querystr+self.subquery.replace("TSVAR", str(end))
+ querystr = self.query.replace("SUBQUERY", querystr)
+ self.all_queries.append(querystr)
+ self.qout_r_args.append("NULL")
+
else:
bgn = int(self.begin_vox)
end = int(self.end_vox)
From noreply at svn.ci.uchicago.edu Thu Jul 23 16:21:18 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Thu, 23 Jul 2009 16:21:18 -0500 (CDT)
Subject: [Swift-commit] r3019 - SwiftApps/SEE/trunk
Message-ID: <20090723212118.6395D9CCBD@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-23 16:21:17 -0500 (Thu, 23 Jul 2009)
New Revision: 3019
Modified:
SwiftApps/SEE/trunk/ampl.swift
SwiftApps/SEE/trunk/runampl.sh
SwiftApps/SEE/trunk/sites_coaster-teraport.xml
SwiftApps/SEE/trunk/swift.properties
SwiftApps/SEE/trunk/tc.data
SwiftApps/SEE/trunk/tgranger-sge-gram2.xml
Log:
Configuration for Ranger coaster debugging
Modified: SwiftApps/SEE/trunk/ampl.swift
===================================================================
--- SwiftApps/SEE/trunk/ampl.swift 2009-07-20 19:25:33 UTC (rev 3018)
+++ SwiftApps/SEE/trunk/ampl.swift 2009-07-23 21:21:17 UTC (rev 3019)
@@ -41,9 +41,9 @@
}
AmplCmd const_cmd ;
-string data_dir = "/home/aespinosa/science/see/run3k";
+string data_dir = "/work/01035/tg802895/science/see/rdraws/all";
-int runs[]=[2001:2002];
+int runs[]=[1:5000];
foreach i in runs {
string instanceID = @strcat("run", i);
Modified: SwiftApps/SEE/trunk/runampl.sh
===================================================================
--- SwiftApps/SEE/trunk/runampl.sh 2009-07-20 19:25:33 UTC (rev 3018)
+++ SwiftApps/SEE/trunk/runampl.sh 2009-07-23 21:21:17 UTC (rev 3019)
@@ -10,3 +10,5 @@
#-tc.file ./tc.data ampl.swift
#swift -runid run3k_firefly -config swift.properties \
#-sites.file ./ff-grid.xml -tc.file ./tc.data ampl.swift
+swift -runid ranger_test -config swift.properties \
+ -sites.file ./tgranger-sge-gram2.xml -tc.file ./tc.data ampl.swift
Modified: SwiftApps/SEE/trunk/sites_coaster-teraport.xml
===================================================================
--- SwiftApps/SEE/trunk/sites_coaster-teraport.xml 2009-07-20 19:25:33 UTC (rev 3018)
+++ SwiftApps/SEE/trunk/sites_coaster-teraport.xml 2009-07-23 21:21:17 UTC (rev 3019)
@@ -1,6 +1,5 @@
-
Modified: SwiftApps/SEE/trunk/swift.properties
===================================================================
--- SwiftApps/SEE/trunk/swift.properties 2009-07-20 19:25:33 UTC (rev 3018)
+++ SwiftApps/SEE/trunk/swift.properties 2009-07-23 21:21:17 UTC (rev 3019)
@@ -244,7 +244,7 @@
# 1 + execution.retries attempts at execution)
#
-execution.retries=1
+execution.retries=0
# Enables/disables replication. Replication is used to deal with jobs sitting
@@ -319,7 +319,7 @@
# Default: 1024
#
-foreach.max.threads=1024
+foreach.max.threads=2048
# controls whether the log file will contain provenance information
# enabling this will increase the size of log files, sometimes
Modified: SwiftApps/SEE/trunk/tc.data
===================================================================
--- SwiftApps/SEE/trunk/tc.data 2009-07-20 19:25:33 UTC (rev 3018)
+++ SwiftApps/SEE/trunk/tc.data 2009-07-23 21:21:17 UTC (rev 3019)
@@ -21,7 +21,7 @@
localhost set1 /home/wilde/angle/data/set1 INSTALLED INTEL32::LINUX null
localhost set3 /home/wilde/angle/data/set3 INSTALLED INTEL32::LINUX null
localhost run_ampl /home/zzhang/SEE/static/run_ampl INSTALLED INTEL32::LINUX null
-tgtacc run_ampl /work/00946/zzhang/SEE/static/run_ampl INSTALLED INTEL32::LINUX null
+RANGER run_ampl /share/home/01035/tg802895/science/see/site_pack/run_ampl INSTALLED INTEL32::LINUX null
ff-grid run_ampl /panfs/panasas/CMS/data/SEE/static/run_ampl INSTALLED INTEL32::LINUX null
TERAPORT run_ampl /home/aespinosa/science/see/SEE/site_pack/run_ampl INSTALLED INTEL32::LINUX null
Modified: SwiftApps/SEE/trunk/tgranger-sge-gram2.xml
===================================================================
--- SwiftApps/SEE/trunk/tgranger-sge-gram2.xml 2009-07-20 19:25:33 UTC (rev 3018)
+++ SwiftApps/SEE/trunk/tgranger-sge-gram2.xml 2009-07-23 21:21:17 UTC (rev 3019)
@@ -1,19 +1,20 @@
-
-
+
TG-CCR080022N
+ normal
+ 10000
+ 10.24
+ 4:00:00
+
+ 16
+ 0.5
+ 50
+ 256
+ 86400
+
+
/scratch/01035/tg802895/see_runs
- 16
- development
- 2
- 10
- 3
- 5
- 5
- 5
- 60
- 120
From noreply at svn.ci.uchicago.edu Sun Jul 26 22:17:04 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Sun, 26 Jul 2009 22:17:04 -0500 (CDT)
Subject: [Swift-commit] r3020 - SwiftApps/SEE/trunk/site_pack
Message-ID: <20090727031704.69BCA9CC8D@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-26 22:17:04 -0500 (Sun, 26 Jul 2009)
New Revision: 3020
Modified:
SwiftApps/SEE/trunk/site_pack/run_ampl
Log:
Mods to extract_expend as requested by Joshua
Modified: SwiftApps/SEE/trunk/site_pack/run_ampl
===================================================================
--- SwiftApps/SEE/trunk/site_pack/run_ampl 2009-07-23 21:21:17 UTC (rev 3019)
+++ SwiftApps/SEE/trunk/site_pack/run_ampl 2009-07-27 03:17:04 UTC (rev 3020)
@@ -207,20 +207,22 @@
# Expend
extract_expend()
{
- echo 'YEAR REGION PROD CON1 '
+ echo 'YEAR REGION CON1 PROD CEM STL NFM MAN'
- egrep '\+|^ *ELC|AGF.*OIL' $1 | sed -e 's/AGF.*OIL.*//' | egrep -v 'LND|LAB|CAP|NAT' |
+ egrep '\+|^ *ELC|AGF.*OIL' $1 | sed -e 's/AGF.*OIL.*//' |
awk '
BEGIN { regnum=0; year=2003 }
NF==1 { region=$1; if ( regnum % 16 == 0 ) year++; regnum++}
- NF!=1 { print year, region, $1, $19 }
+ NF!=1 { print year, region, $19, $1, $8, $9, $10, $11, $7 }
'
}
+#
+
# Price
extract_price()
{
From noreply at svn.ci.uchicago.edu Mon Jul 27 20:14:15 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Mon, 27 Jul 2009 20:14:15 -0500 (CDT)
Subject: [Swift-commit] r3021 - SwiftApps/SEE/trunk/site_pack
Message-ID: <20090728011415.5AEDE9CC9A@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-27 20:14:15 -0500 (Mon, 27 Jul 2009)
New Revision: 3021
Modified:
SwiftApps/SEE/trunk/site_pack/run_ampl
Log:
Fix on run_ampl script
Modified: SwiftApps/SEE/trunk/site_pack/run_ampl
===================================================================
--- SwiftApps/SEE/trunk/site_pack/run_ampl 2009-07-27 03:17:04 UTC (rev 3020)
+++ SwiftApps/SEE/trunk/site_pack/run_ampl 2009-07-28 01:14:15 UTC (rev 3021)
@@ -202,7 +202,7 @@
EOF
}
- Extraction scripts
+# Extraction scripts
# Expend
extract_expend()
From noreply at svn.ci.uchicago.edu Tue Jul 28 10:42:52 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 28 Jul 2009 10:42:52 -0500 (CDT)
Subject: [Swift-commit] r3022 - SwiftApps/SEE/trunk
Message-ID: <20090728154252.399039CC9A@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-28 10:42:51 -0500 (Tue, 28 Jul 2009)
New Revision: 3022
Modified:
SwiftApps/SEE/trunk/ampl.swift
SwiftApps/SEE/trunk/ff-grid.xml
SwiftApps/SEE/trunk/runampl.sh
SwiftApps/SEE/trunk/tc.data
Log:
Update for 7k rdraws_all run
Modified: SwiftApps/SEE/trunk/ampl.swift
===================================================================
--- SwiftApps/SEE/trunk/ampl.swift 2009-07-28 01:14:15 UTC (rev 3021)
+++ SwiftApps/SEE/trunk/ampl.swift 2009-07-28 15:42:51 UTC (rev 3022)
@@ -41,7 +41,7 @@
}
AmplCmd const_cmd ;
-string data_dir = "/work/01035/tg802895/science/see/rdraws/all";
+string data_dir = "/panfs/panasas/CMS/data/SEE/rdraws/all";
int runs[]=[1:5000];
foreach i in runs {
Modified: SwiftApps/SEE/trunk/ff-grid.xml
===================================================================
--- SwiftApps/SEE/trunk/ff-grid.xml 2009-07-28 01:14:15 UTC (rev 3021)
+++ SwiftApps/SEE/trunk/ff-grid.xml 2009-07-28 15:42:51 UTC (rev 3022)
@@ -12,4 +12,16 @@
gt2 ff-grid.unl.edu/jobmanager-pbs
+
+
+
+ 10000.0
+ 10.0
+ /panfs/panasas/CMS/data/SEE/workdir
+ grid
+ default
+ gt2 ff-grid2.unl.edu/jobmanager-pbs
+
+
+
Modified: SwiftApps/SEE/trunk/runampl.sh
===================================================================
--- SwiftApps/SEE/trunk/runampl.sh 2009-07-28 01:14:15 UTC (rev 3021)
+++ SwiftApps/SEE/trunk/runampl.sh 2009-07-28 15:42:51 UTC (rev 3022)
@@ -8,7 +8,7 @@
#-tc.file ./tc.data ampl.swift
#swift -runid teraport -sites.file ./sites-teraport.xml \
#-tc.file ./tc.data ampl.swift
-#swift -runid run3k_firefly -config swift.properties \
- #-sites.file ./ff-grid.xml -tc.file ./tc.data ampl.swift
-swift -runid ranger_test -config swift.properties \
- -sites.file ./tgranger-sge-gram2.xml -tc.file ./tc.data ampl.swift
+swift -runid rdraws-all_firefly -config swift.properties \
+ -sites.file ./ff-grid.xml -tc.file ./tc.data ampl.swift
+#swift -runid ranger_test -config swift.properties \
+ #-sites.file ./tgranger-sge-gram2.xml -tc.file ./tc.data ampl.swift
Modified: SwiftApps/SEE/trunk/tc.data
===================================================================
--- SwiftApps/SEE/trunk/tc.data 2009-07-28 01:14:15 UTC (rev 3021)
+++ SwiftApps/SEE/trunk/tc.data 2009-07-28 15:42:51 UTC (rev 3022)
@@ -22,6 +22,7 @@
localhost set3 /home/wilde/angle/data/set3 INSTALLED INTEL32::LINUX null
localhost run_ampl /home/zzhang/SEE/static/run_ampl INSTALLED INTEL32::LINUX null
RANGER run_ampl /share/home/01035/tg802895/science/see/site_pack/run_ampl INSTALLED INTEL32::LINUX null
-ff-grid run_ampl /panfs/panasas/CMS/data/SEE/static/run_ampl INSTALLED INTEL32::LINUX null
+ff-grid run_ampl /panfs/panasas/CMS/app/SEE/site_pack/run_ampl INSTALLED INTEL32::LINUX null
+ff-grid2 run_ampl /panfs/panasas/CMS/app/SEE/site_pack/run_ampl INSTALLED INTEL32::LINUX null
TERAPORT run_ampl /home/aespinosa/science/see/SEE/site_pack/run_ampl INSTALLED INTEL32::LINUX null
From noreply at svn.ci.uchicago.edu Tue Jul 28 10:42:58 2009
From: noreply at svn.ci.uchicago.edu (noreply at svn.ci.uchicago.edu)
Date: Tue, 28 Jul 2009 10:42:58 -0500 (CDT)
Subject: [Swift-commit] r3023 - SwiftApps/SEE/trunk/site_pack
Message-ID: <20090728154258.7B4A59CC9A@vm-125-59.ci.uchicago.edu>
Author: aespinosa
Date: 2009-07-28 10:42:58 -0500 (Tue, 28 Jul 2009)
New Revision: 3023
Modified:
SwiftApps/SEE/trunk/site_pack/run_ampl
Log:
Bug fix for generating symlinks
Modified: SwiftApps/SEE/trunk/site_pack/run_ampl
===================================================================
--- SwiftApps/SEE/trunk/site_pack/run_ampl 2009-07-28 15:42:51 UTC (rev 3022)
+++ SwiftApps/SEE/trunk/site_pack/run_ampl 2009-07-28 15:42:58 UTC (rev 3023)
@@ -260,52 +260,52 @@
INSTANCE_ID=$2
# Fix missing constants
-if [ ! -a $PREFIX/$INSTANCE_ID/arm_16x16.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/arm_16x16.dat ]; then
ln -sf $SCRIPTDIR/arm_16x16.dat $PREFIX/$INSTANCE_ID/arm_16x16.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/consumersigma.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/consumersigma.dat ]; then
ln -sf $SCRIPTDIR/consumersigma.dat $PREFIX/$INSTANCE_ID/consumersigma.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/producersigma.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/producersigma.dat ]; then
ln -sf $SCRIPTDIR/producersigma.dat $PREFIX/$INSTANCE_ID/producersigma.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/eppa_energy_eff.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/eppa_energy_eff.dat ]; then
ln -sf $SCRIPTDIR/eppa_energy_eff.dat $PREFIX/$INSTANCE_ID/eppa_energy_eff.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/labor_growth.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/labor_growth.dat ]; then
ln -sf $SCRIPTDIR/labor_growth.dat $PREFIX/$INSTANCE_ID/labor_growth.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/consumerdemands.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/consumerdemands.dat ]; then
ln -sf $SCRIPTDIR/consumerdemands.dat $PREFIX/$INSTANCE_ID/consumerdemands.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/consumersalestax.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/consumersalestax.dat ]; then
ln -sf $SCRIPTDIR/consumersalestax.dat $PREFIX/$INSTANCE_ID/consumersalestax.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/consumersalestax.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/consumersalestax.dat ]; then
ln -sf $SCRIPTDIR/consumersalestax.dat $PREFIX/$INSTANCE_ID/consumersalestax.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/export_taxes.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/export_taxes.dat ]; then
ln -sf $SCRIPTDIR/export_taxes.dat $PREFIX/$INSTANCE_ID/export_taxes.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/factor_incomes.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/factor_incomes.dat ]; then
ln -sf $SCRIPTDIR/factor_incomes.dat $PREFIX/$INSTANCE_ID/factor_incomes.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/imports.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/imports.dat ]; then
ln -sf $SCRIPTDIR/imports.dat $PREFIX/$INSTANCE_ID/imports.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/consumerdemands.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/consumerdemands.dat ]; then
ln -sf $SCRIPTDIR/consumerdemands.dat $PREFIX/$INSTANCE_ID/consumerdemands.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/import_taxes.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/import_taxes.dat ]; then
ln -sf $SCRIPTDIR/import_taxes.dat $PREFIX/$INSTANCE_ID/import_taxes.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/otherdemands.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/otherdemands.dat ]; then
ln -sf $SCRIPTDIR/otherdemands.dat $PREFIX/$INSTANCE_ID/otherdemands.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/othersalestax.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/othersalestax.dat ]; then
ln -sf $SCRIPTDIR/othersalestax.dat $PREFIX/$INSTANCE_ID/othersalestax.dat
fi
-if [ ! -a $PREFIX/$INSTANCE_ID/production_taxes.dat ]; then
+if [ ! -f $PREFIX/$INSTANCE_ID/production_taxes.dat ]; then
ln -sf $SCRIPTDIR/production_taxes.dat $PREFIX/$INSTANCE_ID/production_taxes.dat
fi