public class StreamingStep
extends java.lang.Object
See also: Hadoop Streaming
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); AmazonElasticMapReduce emr = new AmazonElasticMapReduceClient(credentials); HadoopJarStepConfig config = new StreamingStep() .withInputs("s3://elasticmapreduce/samples/wordcount/input") .withOutput("s3://my-bucket/output/") .withMapper("s3://elasticmapreduce/samples/wordcount/wordSplitter.py") .withReducer("aggregate") .toHadoopJarStepConfig(); StepConfig wordCount = new StepConfig() .withName("Word Count") .withActionOnFailure("TERMINATE_JOB_FLOW") .withHadoopJarStep(config); RunJobFlowRequest request = new RunJobFlowRequest() .withName("Word Count") .withSteps(wordCount) .withLogUri("s3://log-bucket/") .withInstances(new JobFlowInstancesConfig() .withEc2KeyName("keypairt") .withHadoopVersion("0.20") .withInstanceCount(5) .withKeepJobFlowAliveWhenNoSteps(true) .withMasterInstanceType("m1.small") .withSlaveInstanceType("m1.small")); RunJobFlowResult result = emr.runJobFlow(request);
Modifier and Type | Field and Description |
---|---|
private java.util.Map<java.lang.String,java.lang.String> |
hadoopConfig |
private java.util.List<java.lang.String> |
inputs |
private java.lang.String |
mapper |
private java.lang.String |
output |
private java.lang.String |
reducer |
Constructor and Description |
---|
StreamingStep()
Creates a new default StreamingStep.
|
Modifier and Type | Method and Description |
---|---|
java.util.Map<java.lang.String,java.lang.String> |
getHadoopConfig()
Get the Hadoop config overrides (-D values).
|
java.util.List<java.lang.String> |
getInputs()
Get list of step input paths.
|
java.lang.String |
getMapper()
Get the mapper.
|
java.lang.String |
getOutput()
Get output path.
|
java.lang.String |
getReducer()
Get the reducer
|
void |
setHadoopConfig(java.util.Map<java.lang.String,java.lang.String> hadoopConfig)
Set the Hadoop config overrides (-D values).
|
void |
setInputs(java.util.Collection<java.lang.String> inputs)
Set the list of step input paths.
|
void |
setMapper(java.lang.String mapper)
Set the mapper.
|
void |
setOutput(java.lang.String output)
Set the output path for this step.
|
void |
setReducer(java.lang.String reducer)
Set the reducer
|
HadoopJarStepConfig |
toHadoopJarStepConfig()
Creates the final HadoopJarStepConfig once you are done configuring the step.
|
StreamingStep |
withHadoopConfig(java.lang.String key,
java.lang.String value)
Add a Hadoop config override (-D value).
|
StreamingStep |
withInputs(java.lang.String... inputs)
Add more input paths to this step.
|
StreamingStep |
withMapper(java.lang.String mapper)
Set the mapper
|
StreamingStep |
withOutput(java.lang.String output)
Set the output path for this step.
|
StreamingStep |
withReducer(java.lang.String reducer)
Set the reducer
|
private java.util.List<java.lang.String> inputs
private java.lang.String output
private java.lang.String mapper
private java.lang.String reducer
private java.util.Map<java.lang.String,java.lang.String> hadoopConfig
public java.util.List<java.lang.String> getInputs()
public void setInputs(java.util.Collection<java.lang.String> inputs)
inputs
- List of step inputs.public StreamingStep withInputs(java.lang.String... inputs)
inputs
- A list of inputs to this step.public java.lang.String getOutput()
public void setOutput(java.lang.String output)
output
- Output path.public StreamingStep withOutput(java.lang.String output)
output
- Output pathpublic java.lang.String getMapper()
public void setMapper(java.lang.String mapper)
mapper
- Mapperpublic StreamingStep withMapper(java.lang.String mapper)
mapper
- Mapperpublic java.lang.String getReducer()
public void setReducer(java.lang.String reducer)
reducer
- Reducerpublic StreamingStep withReducer(java.lang.String reducer)
reducer
- Reducerpublic java.util.Map<java.lang.String,java.lang.String> getHadoopConfig()
public void setHadoopConfig(java.util.Map<java.lang.String,java.lang.String> hadoopConfig)
hadoopConfig
- Hadoop config.public StreamingStep withHadoopConfig(java.lang.String key, java.lang.String value)
key
- Hadoop configuration key.value
- Configuration value.public HadoopJarStepConfig toHadoopJarStepConfig()