org.springframework.data.hadoop.batch.spark

Class SparkYarnTasklet

java.lang.Object

org.springframework.data.hadoop.batch.spark.SparkYarnTasklet

All Implemented Interfaces:

org.springframework.batch.core.step.tasklet.Tasklet, org.springframework.batch.core.StepExecutionListener, org.springframework.batch.core.StepListener, org.springframework.beans.factory.InitializingBean

public class SparkYarnTasklet
extends java.lang.Object
implements org.springframework.beans.factory.InitializingBean, org.springframework.batch.core.step.tasklet.Tasklet, org.springframework.batch.core.StepExecutionListener

Spark tasklet running Spark jobs on demand on YARN cluster.

Author:

Thomas Risberg

Constructor Summary

Constructors

Constructor and Description
SparkYarnTasklet()

Method Summary

Methods

Modifier and Type	Method and Description
void	afterPropertiesSet()
org.springframework.batch.core.ExitStatus	afterStep(org.springframework.batch.core.StepExecution stepExecution)
void	beforeStep(org.springframework.batch.core.StepExecution stepExecution)
org.springframework.batch.repeat.RepeatStatus	execute(org.springframework.batch.core.StepContribution contribution, org.springframework.batch.core.scope.context.ChunkContext chunkContext)
void	setAppClass(java.lang.String appClass)
void	setAppJar(java.lang.String appJar)
void	setAppName(java.lang.String appName)
void	setArguments(java.lang.String[] arguments)
void	setDriverCores(java.lang.String driverCores)
void	setDriverMemory(java.lang.String driverMemory)
void	setExecutorCores(java.lang.String executorCores)
void	setExecutorMemory(java.lang.String executorMemory)
void	setExtraClassPath(java.lang.String extraClassPath)
void	setHadoopConfiguration(org.apache.hadoop.conf.Configuration configuration)
void	setNumExecutors(int numExecutors)
void	setResourceArchives(java.lang.String resourceArchives)
void	setResourceFiles(java.lang.String resourceFiles)
void	setSparkAssemblyJar(java.lang.String sparkAssemblyJar)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

SparkYarnTasklet

public SparkYarnTasklet()

Method Detail

execute

public org.springframework.batch.repeat.RepeatStatus execute(org.springframework.batch.core.StepContribution contribution,
                                                    org.springframework.batch.core.scope.context.ChunkContext chunkContext)
                                                      throws java.lang.Exception

Specified by:

execute in interface org.springframework.batch.core.step.tasklet.Tasklet

Throws:

java.lang.Exception

beforeStep

public void beforeStep(org.springframework.batch.core.StepExecution stepExecution)

Specified by:

beforeStep in interface org.springframework.batch.core.StepExecutionListener

afterStep

public org.springframework.batch.core.ExitStatus afterStep(org.springframework.batch.core.StepExecution stepExecution)

Specified by:

afterStep in interface org.springframework.batch.core.StepExecutionListener

afterPropertiesSet

public void afterPropertiesSet()
                        throws java.lang.Exception

Specified by:

afterPropertiesSet in interface org.springframework.beans.factory.InitializingBean

Throws:

java.lang.Exception

setSparkAssemblyJar

public void setSparkAssemblyJar(java.lang.String sparkAssemblyJar)

setExtraClassPath

public void setExtraClassPath(java.lang.String extraClassPath)

setHadoopConfiguration

public void setHadoopConfiguration(org.apache.hadoop.conf.Configuration configuration)

setAppName

public void setAppName(java.lang.String appName)

setAppClass

public void setAppClass(java.lang.String appClass)

setAppJar

public void setAppJar(java.lang.String appJar)

setResourceFiles

public void setResourceFiles(java.lang.String resourceFiles)

setResourceArchives

public void setResourceArchives(java.lang.String resourceArchives)

setExecutorMemory

public void setExecutorMemory(java.lang.String executorMemory)

setExecutorCores

public void setExecutorCores(java.lang.String executorCores)

setDriverMemory

public void setDriverMemory(java.lang.String driverMemory)

setDriverCores

public void setDriverCores(java.lang.String driverCores)

setNumExecutors

public void setNumExecutors(int numExecutors)

setArguments

public void setArguments(java.lang.String[] arguments)