Commit 17518046 authored by Alexander Alexandrov's avatar Alexander Alexandrov
Browse files

Migrated external dependencies to packages that are licensed under Apache 2.0...

Migrated external dependencies to packages that are licensed under Apache 2.0 licenses (commons-cli, jetty).
parent 00d655df
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>eu.stratosphere</groupId>
<artifactId>myriad-driver</artifactId>
<version>0.2.1</version>
<packaging>jar</packaging>
<url>http://maven.apache.org</url>
<properties>
......@@ -67,14 +68,16 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.martiansoftware</groupId>
<artifactId>jsap</artifactId>
<version>2.1</version>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.2</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.sun.net.httpserver</groupId>
<artifactId>http</artifactId>
<version>20070405</version>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<version>6.1.26</version>
<scope>provided</scope>
</dependency>
</dependencies>
</dependencies>
</project>
......@@ -14,21 +14,21 @@
**********************************************************************************************************************/
package eu.stratosphere.myriad.driver;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Iterator;
import com.martiansoftware.jsap.FlaggedOption;
import com.martiansoftware.jsap.JSAP;
import com.martiansoftware.jsap.JSAPException;
import com.martiansoftware.jsap.JSAPResult;
import com.martiansoftware.jsap.UnflaggedOption;
import com.martiansoftware.jsap.stringparsers.DoubleStringParser;
import com.martiansoftware.jsap.stringparsers.FileStringParser;
import com.martiansoftware.jsap.stringparsers.ShortStringParser;
import com.martiansoftware.jsap.stringparsers.StringStringParser;
import java.util.Arrays;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import eu.stratosphere.myriad.driver.hadoop.MyriadDriverHadoopJob;
import eu.stratosphere.myriad.driver.parameters.DriverJobParameters;
......@@ -40,91 +40,58 @@ import eu.stratosphere.myriad.driver.parameters.DriverJobParametersFamily;
*/
public class MyriadDriverFrontend {
private JSAP optionsParser = new JSAP();
private final Options options;
public MyriadDriverFrontend() {
initialize();
}
/**
*
*/
private void initialize() {
try {
// dgen-install-dir
UnflaggedOption optDGenInstallDir = new UnflaggedOption("dgen-install-dir");
optDGenInstallDir.setStringParser(FileStringParser.getParser());
optDGenInstallDir.setRequired(true);
optDGenInstallDir.setHelp("absolute data generator installation directory");
this.optionsParser.registerParameter(optDGenInstallDir);
// scaling-factor
FlaggedOption optScalingFactor = new FlaggedOption("scaling-factor");
optScalingFactor.setShortFlag('s');
optScalingFactor.setLongFlag("scaling-factor");
optScalingFactor.setStringParser(DoubleStringParser.getParser());
optScalingFactor.setRequired(true);
optScalingFactor.setDefault("1.0");
optScalingFactor.setHelp("scaling factor (s=1 generates 1GB)");
this.optionsParser.registerParameter(optScalingFactor);
// dataset-id
FlaggedOption optDatasetID = new FlaggedOption("dataset-id");
optDatasetID.setShortFlag('m');
optDatasetID.setLongFlag("dataset-id");
optDatasetID.setStringParser(StringStringParser.getParser());
optDatasetID.setRequired(true);
optDatasetID.setDefault("default-dataset");
optDatasetID.setHelp("ID of the generated Myriad dataset");
this.optionsParser.registerParameter(optDatasetID);
// node-count
FlaggedOption optNodeCount = new FlaggedOption("node-count");
optNodeCount.setShortFlag('N');
optNodeCount.setLongFlag("node-count");
optNodeCount.setStringParser(ShortStringParser.getParser());
optNodeCount.setRequired(true);
optNodeCount.setDefault("1");
optNodeCount.setHelp("degree of parallelism (i.e. total number of partitions)");
this.optionsParser.registerParameter(optNodeCount);
// output-base
FlaggedOption optOutputBase = new FlaggedOption("output-base");
optOutputBase.setShortFlag('o');
optOutputBase.setLongFlag("output-base");
optOutputBase.setStringParser(FileStringParser.getParser());
optOutputBase.setRequired(true);
optOutputBase.setDefault("/tmp");
optOutputBase.setHelp("base path for writing the output");
this.optionsParser.registerParameter(optOutputBase);
// execute-stages
FlaggedOption optExecuteStages = new FlaggedOption("execute-stage");
optExecuteStages.setShortFlag('x');
optExecuteStages.setLongFlag("execute-stage");
optExecuteStages.setStringParser(StringStringParser.getParser());
optExecuteStages.setRequired(true);
optExecuteStages.setList(true);
optExecuteStages.setListSeparator(',');
optExecuteStages.setHelp("specify stages to be executed");
this.optionsParser.registerParameter(optExecuteStages);
} catch (JSAPException e) {
System.err.println("Could not construct JSAP options: " + e.getMessage());
System.exit(1);
}
// dgen-install-dir
this.options = new Options();
// scaling-factor
OptionBuilder.hasArg();
OptionBuilder.withArgName("double");
OptionBuilder.withDescription("scaling factor (s=1 generates 1GB)");
OptionBuilder.withLongOpt("scaling-factor");
this.options.addOption(OptionBuilder.create('s'));
// dataset-id
OptionBuilder.hasArg();
OptionBuilder.withArgName("string");
OptionBuilder.withDescription("ID of the generated Myriad dataset");
OptionBuilder.withLongOpt("dataset-id");
this.options.addOption(OptionBuilder.create('m'));
// node-count
OptionBuilder.hasArg();
OptionBuilder.withArgName("int");
OptionBuilder.withDescription("degree of parallelism (i.e. total number of partitions)");
OptionBuilder.withArgName("node-count");
this.options.addOption(OptionBuilder.create('N'));
// output-base
OptionBuilder.hasArg();
OptionBuilder.withArgName("path");
OptionBuilder.withDescription("base path for writing the output");
OptionBuilder.withLongOpt("output-base");
this.options.addOption(OptionBuilder.create('o'));
// execute-stages
OptionBuilder.hasArgs();
OptionBuilder.withArgName("stagename");
OptionBuilder.withDescription("specify specific stages to be executed");
OptionBuilder.withLongOpt("execute-stage");
this.options.addOption(OptionBuilder.create('x'));
}
/**
* @param args
*/
private void process(String[] args) throws ParseOptionsException {
private void process(String[] args) throws ParsedOptionsException, ParseException {
// parse options
JSAPResult parsedOptions = this.optionsParser.parse(args);
ParsedOptions parsedOptions = this.parseOptions(args);
// throw exception on parse failure
if (!parsedOptions.success()) {
throw new ParseOptionsException(parsedOptions);
throw new ParsedOptionsException(parsedOptions);
}
try {
......@@ -146,6 +113,50 @@ public class MyriadDriverFrontend {
}
/**
* @param args
* @return
*/
private ParsedOptions parseOptions(String[] args) throws ParseException {
ParsedOptions parsedOptions = new ParsedOptions();
if (args.length < 1) {
throw new ParseException("Missing dgen-install-dir argument");
}
parsedOptions.setFile("dgen-install-dir", new File(args[0]));
// parse the command line arguments
CommandLineParser parser = new PosixParser();
CommandLine line = parser.parse(this.options, Arrays.copyOfRange(args, 1, args.length));
if (line.hasOption('x')) {
parsedOptions.setStringArray("execute-stage", line.getOptionValues('x'));
} else {
parsedOptions.setErrorMessage("execute-stage",
"You should provide at least one data generator stage to be executed");
}
try {
parsedOptions.setFloat("scaling-factor", Float.parseFloat(line.getOptionValue('s', "1.0")));
} catch (NumberFormatException e) {
parsedOptions.setErrorMessage("scaling-factor", e.getMessage());
}
try {
parsedOptions.setShort("node-count", Short.parseShort(line.getOptionValue('N', "1")));
} catch (NumberFormatException e) {
parsedOptions.setErrorMessage("node-count", e.getMessage());
}
parsedOptions.setString("dataset-id", line.getOptionValue('m', "default-dataset"));
parsedOptions.setFile("output-base", new File(line.getOptionValue('o', "/tmp")));
return parsedOptions;
}
/**
* Factory method.
*
* @param parameters
* @return
*/
......@@ -153,22 +164,28 @@ public class MyriadDriverFrontend {
return new MyriadDriverHadoopJob(parameters);
}
private void printErrors(PrintStream out, JSAPResult parsedOptions) {
@SuppressWarnings("rawtypes")
Iterator errs = parsedOptions.getErrorMessageIterator();
while (errs.hasNext()) {
out.println("Error: " + errs.next());
private void printErrors(PrintStream out, ParsedOptions parsedOptions) {
for (String message : parsedOptions.getErrorMessages()) {
out.println("Error: " + message);
}
out.println();
}
private void printUsage(PrintStream out) {
out.println("Usage: (hadoop|pact-client) jar myriad-driver-jobs.jar <dgen-install-dir> [OPTIONS]");
out.println();
private String getUsage() {
return "(hadoop|pact-client) jar myriad-driver-jobs.jar <dgen-dir> [OPTIONS]";
}
private void printHelp(PrintStream out) {
out.println(this.optionsParser.getHelp());
PrintWriter pw = new PrintWriter(out);
int width = 80;
int leftPad = 0;
int descPad = 2;
HelpFormatter formatter = new HelpFormatter();
formatter.setSyntaxPrefix("Usage: ");
formatter.printHelp(pw, width, getUsage(), "\nAvailable Options:\n \n", this.options, leftPad, descPad, null, false);
pw.flush();
}
public static void main(String[] args) {
......@@ -176,9 +193,13 @@ public class MyriadDriverFrontend {
try {
frontend.process(args);
} catch (ParseOptionsException e) {
} catch (ParseException e) {
System.err.println("Could not parse command line string");
System.err.println();
frontend.printHelp(System.err);
System.exit(1);
} catch (ParsedOptionsException e) {
frontend.printErrors(System.err, e.getParsedOptions());
frontend.printUsage(System.err);
frontend.printHelp(System.err);
System.exit(1);
}
......
/***********************************************************************************************************************
*
* Copyright (C) 2010 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*
**********************************************************************************************************************/
package eu.stratosphere.myriad.driver;
import java.io.File;
import java.util.Collection;
import java.util.HashMap;
/**
* @author Alexander Alexandrov (alexander.alexandrov@tu-berlin.de)
*/
public class ParsedOptions {
private final HashMap<String, String> stringOptions;
private final HashMap<String, File> fileOptions;
private final HashMap<String, Float> floatOptions;
private final HashMap<String, Short> shortOptions;
private final HashMap<String, String[]> stringArrayOptions;
private final HashMap<String, File[]> fileArrayOptions;
private final HashMap<String, Float[]> floatArrayOptions;
private final HashMap<String, Short[]> shortArrayOptions;
private final HashMap<String, String> errorMessages;
public ParsedOptions() {
// init calar option containers
this.stringOptions = new HashMap<String, String>();
this.fileOptions = new HashMap<String, File>();
this.floatOptions = new HashMap<String, Float>();
this.shortOptions = new HashMap<String, Short>();
// init array option containers
this.stringArrayOptions = new HashMap<String, String[]>();
this.fileArrayOptions = new HashMap<String, File[]>();
this.floatArrayOptions = new HashMap<String, Float[]>();
this.shortArrayOptions = new HashMap<String, Short[]>();
this.errorMessages = new HashMap<String, String>();
}
/**
* @return
*/
public boolean success() {
return this.errorMessages.isEmpty();
}
/**
* @param string
* @param message
*/
public void setErrorMessage(String optionKey, String message) {
this.errorMessages.put(optionKey, message);
}
/**
* @return
*/
public Collection<String> getErrorMessages() {
return this.errorMessages.values();
}
/**
* @param optionKey
* @return
*/
public String getString(String optionKey) {
return this.stringOptions.get(optionKey);
}
/**
* @param optionKey
* @param option
*/
public void setString(String optionKey, String option) {
this.stringOptions.put(optionKey, option);
}
/**
* @param optionKey
* @return
*/
public File getFile(String optionKey) {
return this.fileOptions.get(optionKey);
}
/**
* @param optionKey
* @param option
*/
public void setFile(String optionKey, File option) {
this.fileOptions.put(optionKey, option);
}
/**
* @param optionKey
* @return
*/
public Float getFloat(String optionKey) {
return this.floatOptions.get(optionKey);
}
/**
* @param optionKey
* @param option
*/
public void setFloat(String optionKey, Float option) {
this.floatOptions.put(optionKey, option);
}
/**
* @param optionKey
* @return
*/
public Short getShort(String optionKey) {
return this.shortOptions.get(optionKey);
}
/**
* @param optionKey
* @param option
*/
public void setShort(String optionKey, Short option) {
this.shortOptions.put(optionKey, option);
}
/**
* @param optionKey
* @return
*/
public String[] getStringArray(String optionKey) {
return this.stringArrayOptions.get(optionKey);
}
/**
* @param optionKey
* @param option
*/
public void setStringArray(String optionKey, String[] option) {
this.stringArrayOptions.put(optionKey, option);
}
/**
* @param optionKey
* @return
*/
public File[] getFileArray(String optionKey) {
return this.fileArrayOptions.get(optionKey);
}
/**
* @param optionKey
* @param option
*/
public void setFileArray(String optionKey, File[] option) {
this.fileArrayOptions.put(optionKey, option);
}
/**
* @param optionKey
* @return
*/
public Float[] getFloatList(String optionKey) {
return this.floatArrayOptions.get(optionKey);
}
/**
* @param optionKey
* @param option
*/
public void setFloatArray(String optionKey, Float[] option) {
this.floatArrayOptions.put(optionKey, option);
}
/**
* @param optionKey
* @return
*/
public Short[] getShortList(String optionKey) {
return this.shortArrayOptions.get(optionKey);
}
/**
* @param optionKey
* @param option
*/
public void setShortArray(String optionKey, Short[] option) {
this.shortArrayOptions.put(optionKey, option);
}
}
......@@ -14,19 +14,17 @@
**********************************************************************************************************************/
package eu.stratosphere.myriad.driver;
import com.martiansoftware.jsap.JSAPResult;
public class ParseOptionsException extends Exception {
public class ParsedOptionsException extends Exception {
private static final long serialVersionUID = -250326135439737607L;
private final JSAPResult parsedOptions;
private final ParsedOptions parsedOptions;
public ParseOptionsException(JSAPResult parsedOptions) {
public ParsedOptionsException(ParsedOptions parsedOptions) {
this.parsedOptions = parsedOptions;
}
public JSAPResult getParsedOptions() {
public ParsedOptions getParsedOptions() {
return this.parsedOptions;
}
}
\ No newline at end of file
......@@ -65,12 +65,12 @@ public class MyriadInputFormat implements InputFormat<NullWritable, Text> {
return nodePath;
}
public static void setScalingFactor(JobConf conf, double scalingFactor) {
conf.setFloat("mapred.myriad.dgen.scaling.factor", (float) scalingFactor);
public static void setScalingFactor(JobConf conf, float scalingFactor) {
conf.setFloat("mapred.myriad.dgen.scaling.factor", scalingFactor);
}
public static double getScalingFactor(JobConf conf) {
double scalingFactor = conf.getFloat("mapred.myriad.dgen.scaling.factor", -1);
public static float getScalingFactor(JobConf conf) {
float scalingFactor = conf.getFloat("mapred.myriad.dgen.scaling.factor", -1);
if (scalingFactor <= 0) {
throw new IllegalArgumentException("Bad `mapred.myriad.dgen.scaling.factor` parameter value");
}
......@@ -147,8 +147,8 @@ public class MyriadInputFormat implements InputFormat<NullWritable, Text> {
File outputBase = new File(MyriadInputFormat.getOutputBase(conf));
String datasetID = MyriadInputFormat.getDatasetID(conf);
String stage = MyriadInputFormat.getStage(conf);
double scalingFactor = MyriadInputFormat.getScalingFactor(conf);
float scalingFactor = MyriadInputFormat.getScalingFactor(conf);
short nodeCount = (short) MyriadInputFormat.getNodeCount(conf);
return new SocketReaderParameters(dgenInstallDir , outputBase, datasetID, stage, scalingFactor, nodeCount, nodeID);
return new SocketReaderParameters(dgenInstallDir, outputBase, datasetID, stage, scalingFactor, nodeCount, nodeID);
}
}
\ No newline at end of file
......@@ -38,14 +38,14 @@ public class DriverJobParameters {
private final String stage;
private final double scalingFactor;
private final float scalingFactor;
private final short nodeCount;
private final Properties dgenNodeProperties;
public DriverJobParameters(File dgenInstallDir, File outputBase, String datasetID, String stage,
double scalingFactor, short nodeCount) throws DriverJobParametersException {
float scalingFactor, short nodeCount) throws DriverJobParametersException {
this.dgenInstallDir = dgenInstallDir;
this.dgenName = this.dgenInstallDir.getName();