org.apache.spark.util

Class JsonProtocol

Object

org.apache.spark.util.JsonProtocol

public class JsonProtocol
extends Object

Serializes SparkListener events to/from JSON. This protocol provides strong backwards- and forwards-compatibility guarantees: any version of Spark should be able to read JSON output written by any other version, including newer versions.

JsonProtocolSuite contains backwards-compatibility tests which check that the current version of JsonProtocol is able to read output written by earlier versions. We do not currently have tests for reading newer JSON output with older Spark versions.

To ensure that we provide these guarantees, follow these rules when modifying these methods:

- Never delete any JSON fields. - Any new JSON fields should be optional; use jsonOption when reading these fields in *FromJson methods.

Constructor Summary

Constructors

Constructor and Description
JsonProtocol()

Method Summary

Modifier and Type	Method and Description
static AccumulableInfo	accumulableInfoFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	accumulableInfoToJson(AccumulableInfo accumulableInfo)
static org.json4s.JsonAST.JArray	accumulablesToJson(scala.collection.Iterable<AccumulableInfo> accumulables)
static SparkListenerApplicationEnd	applicationEndFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	applicationEndToJson(SparkListenerApplicationEnd applicationEnd)
static SparkListenerApplicationStart	applicationStartFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	applicationStartToJson(SparkListenerApplicationStart applicationStart)
static SparkListenerBlockManagerAdded	blockManagerAddedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockManagerAddedToJson(SparkListenerBlockManagerAdded blockManagerAdded)
static BlockManagerId	blockManagerIdFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockManagerIdToJson(BlockManagerId blockManagerId)
static SparkListenerBlockManagerRemoved	blockManagerRemovedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockManagerRemovedToJson(SparkListenerBlockManagerRemoved blockManagerRemoved)
static BlockStatus	blockStatusFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockStatusToJson(BlockStatus blockStatus)
static BlockUpdatedInfo	blockUpdatedInfoFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockUpdatedInfoToJson(BlockUpdatedInfo blockUpdatedInfo)
static SparkListenerBlockUpdated	blockUpdateFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	blockUpdateToJson(SparkListenerBlockUpdated blockUpdate)
static SparkListenerEnvironmentUpdate	environmentUpdateFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	environmentUpdateToJson(SparkListenerEnvironmentUpdate environmentUpdate)
static Exception	exceptionFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	exceptionToJson(Exception exception)
static SparkListenerExecutorAdded	executorAddedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	executorAddedToJson(SparkListenerExecutorAdded executorAdded)
static ExecutorInfo	executorInfoFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	executorInfoToJson(ExecutorInfo executorInfo)
static org.apache.spark.executor.ExecutorMetrics	executorMetricsFromJson(org.json4s.JsonAST.JValue json) Extract the executor metrics from JSON.
static org.json4s.JsonAST.JValue	executorMetricsToJson(org.apache.spark.executor.ExecutorMetrics executorMetrics) Convert executor metrics to JSON.
static SparkListenerExecutorMetricsUpdate	executorMetricsUpdateFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	executorMetricsUpdateToJson(SparkListenerExecutorMetricsUpdate metricsUpdate)
static SparkListenerExecutorRemoved	executorRemovedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	executorRemovedToJson(SparkListenerExecutorRemoved executorRemoved)
static ExecutorResourceRequest	executorResourceRequestFromJson(org.json4s.JsonAST.JValue json)
static scala.collection.Map<String,ExecutorResourceRequest>	executorResourceRequestMapFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	executorResourceRequestMapToJson(scala.collection.Map<String,ExecutorResourceRequest> m)
static org.json4s.JsonAST.JValue	executorResourceRequestToJson(ExecutorResourceRequest execReq)
static SparkListenerJobEnd	jobEndFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	jobEndToJson(SparkListenerJobEnd jobEnd)
static JobResult	jobResultFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	jobResultToJson(JobResult jobResult)
static SparkListenerJobStart	jobStartFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	jobStartToJson(SparkListenerJobStart jobStart)
static SparkListenerLogStart	logStartFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	logStartToJson(SparkListenerLogStart logStart)
static scala.collection.Map<String,String>	mapFromJson(org.json4s.JsonAST.JValue json) -------------------------------- * Util JSON deserialization methods |
static org.json4s.JsonAST.JValue	mapToJson(scala.collection.Map<String,String> m) ------------------------------ * Util JSON serialization methods |
static java.util.Properties	propertiesFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	propertiesToJson(java.util.Properties properties)
static RDDInfo	rddInfoFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	rddInfoToJson(RDDInfo rddInfo)
static SparkListenerResourceProfileAdded	resourceProfileAddedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	resourceProfileAddedToJson(SparkListenerResourceProfileAdded profileAdded)
static scala.collection.Map<String,ResourceInformation>	resourcesMapFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	resourcesMapToJson(scala.collection.Map<String,ResourceInformation> m)
static SparkListenerEvent	sparkEventFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	sparkEventToJson(SparkListenerEvent event) ------------------------------------------------- * JSON serialization methods for SparkListenerEvents |
static StackTraceElement[]	stackTraceFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	stackTraceToJson(StackTraceElement[] stackTrace)
static SparkListenerStageCompleted	stageCompletedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	stageCompletedToJson(SparkListenerStageCompleted stageCompleted)
static SparkListenerStageExecutorMetrics	stageExecutorMetricsFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	stageExecutorMetricsToJson(SparkListenerStageExecutorMetrics metrics)
static StageInfo	stageInfoFromJson(org.json4s.JsonAST.JValue json) --------------------------------------------------------------------- * JSON deserialization methods for classes SparkListenerEvents depend on |
static org.json4s.JsonAST.JValue	stageInfoToJson(StageInfo stageInfo) ------------------------------------------------------------------- * JSON serialization methods for classes SparkListenerEvents depend on |
static SparkListenerStageSubmitted	stageSubmittedFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	stageSubmittedToJson(SparkListenerStageSubmitted stageSubmitted)
static StorageLevel	storageLevelFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	storageLevelToJson(StorageLevel storageLevel)
static SparkListenerTaskEnd	taskEndFromJson(org.json4s.JsonAST.JValue json)
static TaskEndReason	taskEndReasonFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskEndReasonToJson(TaskEndReason taskEndReason)
static org.json4s.JsonAST.JValue	taskEndToJson(SparkListenerTaskEnd taskEnd)
static SparkListenerTaskGettingResult	taskGettingResultFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskGettingResultToJson(SparkListenerTaskGettingResult taskGettingResult)
static TaskInfo	taskInfoFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskInfoToJson(TaskInfo taskInfo)
static org.apache.spark.executor.TaskMetrics	taskMetricsFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskMetricsToJson(org.apache.spark.executor.TaskMetrics taskMetrics)
static TaskResourceRequest	taskResourceRequestFromJson(org.json4s.JsonAST.JValue json)
static scala.collection.Map<String,TaskResourceRequest>	taskResourceRequestMapFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskResourceRequestMapToJson(scala.collection.Map<String,TaskResourceRequest> m)
static org.json4s.JsonAST.JValue	taskResourceRequestToJson(TaskResourceRequest taskReq)
static SparkListenerTaskStart	taskStartFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	taskStartToJson(SparkListenerTaskStart taskStart)
static SparkListenerUnpersistRDD	unpersistRDDFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	unpersistRDDToJson(SparkListenerUnpersistRDD unpersistRDD)
static java.util.UUID	UUIDFromJson(org.json4s.JsonAST.JValue json)
static org.json4s.JsonAST.JValue	UUIDToJson(java.util.UUID id)

Methods inherited from class Object

equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

JsonProtocol

public JsonProtocol()

Method Detail

sparkEventToJson

public static org.json4s.JsonAST.JValue sparkEventToJson(SparkListenerEvent event)

------------------------------------------------- * JSON serialization methods for SparkListenerEvents |

Parameters:

event - (undocumented)

Returns:

(undocumented) --------------------------------------------------

stageSubmittedToJson

public static org.json4s.JsonAST.JValue stageSubmittedToJson(SparkListenerStageSubmitted stageSubmitted)

stageCompletedToJson

public static org.json4s.JsonAST.JValue stageCompletedToJson(SparkListenerStageCompleted stageCompleted)

taskStartToJson

public static org.json4s.JsonAST.JValue taskStartToJson(SparkListenerTaskStart taskStart)

taskGettingResultToJson

public static org.json4s.JsonAST.JValue taskGettingResultToJson(SparkListenerTaskGettingResult taskGettingResult)

taskEndToJson

public static org.json4s.JsonAST.JValue taskEndToJson(SparkListenerTaskEnd taskEnd)

jobStartToJson

public static org.json4s.JsonAST.JValue jobStartToJson(SparkListenerJobStart jobStart)

jobEndToJson

public static org.json4s.JsonAST.JValue jobEndToJson(SparkListenerJobEnd jobEnd)

environmentUpdateToJson

public static org.json4s.JsonAST.JValue environmentUpdateToJson(SparkListenerEnvironmentUpdate environmentUpdate)

blockManagerAddedToJson

public static org.json4s.JsonAST.JValue blockManagerAddedToJson(SparkListenerBlockManagerAdded blockManagerAdded)

blockManagerRemovedToJson

public static org.json4s.JsonAST.JValue blockManagerRemovedToJson(SparkListenerBlockManagerRemoved blockManagerRemoved)

unpersistRDDToJson

public static org.json4s.JsonAST.JValue unpersistRDDToJson(SparkListenerUnpersistRDD unpersistRDD)

applicationStartToJson

public static org.json4s.JsonAST.JValue applicationStartToJson(SparkListenerApplicationStart applicationStart)

applicationEndToJson

public static org.json4s.JsonAST.JValue applicationEndToJson(SparkListenerApplicationEnd applicationEnd)

resourceProfileAddedToJson

public static org.json4s.JsonAST.JValue resourceProfileAddedToJson(SparkListenerResourceProfileAdded profileAdded)

executorAddedToJson

public static org.json4s.JsonAST.JValue executorAddedToJson(SparkListenerExecutorAdded executorAdded)

executorRemovedToJson

public static org.json4s.JsonAST.JValue executorRemovedToJson(SparkListenerExecutorRemoved executorRemoved)

logStartToJson

public static org.json4s.JsonAST.JValue logStartToJson(SparkListenerLogStart logStart)

executorMetricsUpdateToJson

public static org.json4s.JsonAST.JValue executorMetricsUpdateToJson(SparkListenerExecutorMetricsUpdate metricsUpdate)

stageExecutorMetricsToJson

public static org.json4s.JsonAST.JValue stageExecutorMetricsToJson(SparkListenerStageExecutorMetrics metrics)

blockUpdateToJson

public static org.json4s.JsonAST.JValue blockUpdateToJson(SparkListenerBlockUpdated blockUpdate)

stageInfoToJson

public static org.json4s.JsonAST.JValue stageInfoToJson(StageInfo stageInfo)

------------------------------------------------------------------- * JSON serialization methods for classes SparkListenerEvents depend on |

Parameters:

stageInfo - (undocumented)

Returns:

(undocumented) --------------------------------------------------------------------

taskInfoToJson

public static org.json4s.JsonAST.JValue taskInfoToJson(TaskInfo taskInfo)

accumulablesToJson

public static org.json4s.JsonAST.JArray accumulablesToJson(scala.collection.Iterable<AccumulableInfo> accumulables)

accumulableInfoToJson

public static org.json4s.JsonAST.JValue accumulableInfoToJson(AccumulableInfo accumulableInfo)

taskMetricsToJson

public static org.json4s.JsonAST.JValue taskMetricsToJson(org.apache.spark.executor.TaskMetrics taskMetrics)

executorMetricsToJson

public static org.json4s.JsonAST.JValue executorMetricsToJson(org.apache.spark.executor.ExecutorMetrics executorMetrics)

Convert executor metrics to JSON.

taskEndReasonToJson

public static org.json4s.JsonAST.JValue taskEndReasonToJson(TaskEndReason taskEndReason)

blockManagerIdToJson

public static org.json4s.JsonAST.JValue blockManagerIdToJson(BlockManagerId blockManagerId)

jobResultToJson

public static org.json4s.JsonAST.JValue jobResultToJson(JobResult jobResult)

rddInfoToJson

public static org.json4s.JsonAST.JValue rddInfoToJson(RDDInfo rddInfo)

storageLevelToJson

public static org.json4s.JsonAST.JValue storageLevelToJson(StorageLevel storageLevel)

blockStatusToJson

public static org.json4s.JsonAST.JValue blockStatusToJson(BlockStatus blockStatus)

executorInfoToJson

public static org.json4s.JsonAST.JValue executorInfoToJson(ExecutorInfo executorInfo)

resourcesMapToJson

public static org.json4s.JsonAST.JValue resourcesMapToJson(scala.collection.Map<String,ResourceInformation> m)

blockUpdatedInfoToJson

public static org.json4s.JsonAST.JValue blockUpdatedInfoToJson(BlockUpdatedInfo blockUpdatedInfo)

executorResourceRequestToJson

public static org.json4s.JsonAST.JValue executorResourceRequestToJson(ExecutorResourceRequest execReq)

executorResourceRequestMapToJson

public static org.json4s.JsonAST.JValue executorResourceRequestMapToJson(scala.collection.Map<String,ExecutorResourceRequest> m)

taskResourceRequestToJson

public static org.json4s.JsonAST.JValue taskResourceRequestToJson(TaskResourceRequest taskReq)

taskResourceRequestMapToJson

public static org.json4s.JsonAST.JValue taskResourceRequestMapToJson(scala.collection.Map<String,TaskResourceRequest> m)

mapToJson

public static org.json4s.JsonAST.JValue mapToJson(scala.collection.Map<String,String> m)

------------------------------ * Util JSON serialization methods |

Parameters:

m - (undocumented)

Returns:

(undocumented) -------------------------------

propertiesToJson

public static org.json4s.JsonAST.JValue propertiesToJson(java.util.Properties properties)

UUIDToJson

public static org.json4s.JsonAST.JValue UUIDToJson(java.util.UUID id)

stackTraceToJson

public static org.json4s.JsonAST.JValue stackTraceToJson(StackTraceElement[] stackTrace)

exceptionToJson

public static org.json4s.JsonAST.JValue exceptionToJson(Exception exception)

sparkEventFromJson

public static SparkListenerEvent sparkEventFromJson(org.json4s.JsonAST.JValue json)

stageSubmittedFromJson

public static SparkListenerStageSubmitted stageSubmittedFromJson(org.json4s.JsonAST.JValue json)

stageCompletedFromJson

public static SparkListenerStageCompleted stageCompletedFromJson(org.json4s.JsonAST.JValue json)

taskStartFromJson

public static SparkListenerTaskStart taskStartFromJson(org.json4s.JsonAST.JValue json)

taskGettingResultFromJson

public static SparkListenerTaskGettingResult taskGettingResultFromJson(org.json4s.JsonAST.JValue json)

executorMetricsFromJson

public static org.apache.spark.executor.ExecutorMetrics executorMetricsFromJson(org.json4s.JsonAST.JValue json)

Extract the executor metrics from JSON.

taskEndFromJson

public static SparkListenerTaskEnd taskEndFromJson(org.json4s.JsonAST.JValue json)

jobStartFromJson

public static SparkListenerJobStart jobStartFromJson(org.json4s.JsonAST.JValue json)

jobEndFromJson

public static SparkListenerJobEnd jobEndFromJson(org.json4s.JsonAST.JValue json)

resourceProfileAddedFromJson

public static SparkListenerResourceProfileAdded resourceProfileAddedFromJson(org.json4s.JsonAST.JValue json)

executorResourceRequestFromJson

public static ExecutorResourceRequest executorResourceRequestFromJson(org.json4s.JsonAST.JValue json)

taskResourceRequestFromJson

public static TaskResourceRequest taskResourceRequestFromJson(org.json4s.JsonAST.JValue json)

taskResourceRequestMapFromJson

public static scala.collection.Map<String,TaskResourceRequest> taskResourceRequestMapFromJson(org.json4s.JsonAST.JValue json)

executorResourceRequestMapFromJson

public static scala.collection.Map<String,ExecutorResourceRequest> executorResourceRequestMapFromJson(org.json4s.JsonAST.JValue json)

environmentUpdateFromJson

public static SparkListenerEnvironmentUpdate environmentUpdateFromJson(org.json4s.JsonAST.JValue json)

blockManagerAddedFromJson

public static SparkListenerBlockManagerAdded blockManagerAddedFromJson(org.json4s.JsonAST.JValue json)

blockManagerRemovedFromJson

public static SparkListenerBlockManagerRemoved blockManagerRemovedFromJson(org.json4s.JsonAST.JValue json)

unpersistRDDFromJson

public static SparkListenerUnpersistRDD unpersistRDDFromJson(org.json4s.JsonAST.JValue json)

applicationStartFromJson

public static SparkListenerApplicationStart applicationStartFromJson(org.json4s.JsonAST.JValue json)

applicationEndFromJson

public static SparkListenerApplicationEnd applicationEndFromJson(org.json4s.JsonAST.JValue json)

executorAddedFromJson

public static SparkListenerExecutorAdded executorAddedFromJson(org.json4s.JsonAST.JValue json)

executorRemovedFromJson

public static SparkListenerExecutorRemoved executorRemovedFromJson(org.json4s.JsonAST.JValue json)

logStartFromJson

public static SparkListenerLogStart logStartFromJson(org.json4s.JsonAST.JValue json)

executorMetricsUpdateFromJson

public static SparkListenerExecutorMetricsUpdate executorMetricsUpdateFromJson(org.json4s.JsonAST.JValue json)

stageExecutorMetricsFromJson

public static SparkListenerStageExecutorMetrics stageExecutorMetricsFromJson(org.json4s.JsonAST.JValue json)

blockUpdateFromJson

public static SparkListenerBlockUpdated blockUpdateFromJson(org.json4s.JsonAST.JValue json)

stageInfoFromJson

public static StageInfo stageInfoFromJson(org.json4s.JsonAST.JValue json)

--------------------------------------------------------------------- * JSON deserialization methods for classes SparkListenerEvents depend on |

Parameters:

json - (undocumented)

Returns:

(undocumented) ----------------------------------------------------------------------

taskInfoFromJson

public static TaskInfo taskInfoFromJson(org.json4s.JsonAST.JValue json)

accumulableInfoFromJson

public static AccumulableInfo accumulableInfoFromJson(org.json4s.JsonAST.JValue json)

taskMetricsFromJson

public static org.apache.spark.executor.TaskMetrics taskMetricsFromJson(org.json4s.JsonAST.JValue json)

taskEndReasonFromJson

public static TaskEndReason taskEndReasonFromJson(org.json4s.JsonAST.JValue json)

blockManagerIdFromJson

public static BlockManagerId blockManagerIdFromJson(org.json4s.JsonAST.JValue json)

jobResultFromJson

public static JobResult jobResultFromJson(org.json4s.JsonAST.JValue json)

rddInfoFromJson

public static RDDInfo rddInfoFromJson(org.json4s.JsonAST.JValue json)

storageLevelFromJson

public static StorageLevel storageLevelFromJson(org.json4s.JsonAST.JValue json)

blockStatusFromJson

public static BlockStatus blockStatusFromJson(org.json4s.JsonAST.JValue json)

executorInfoFromJson

public static ExecutorInfo executorInfoFromJson(org.json4s.JsonAST.JValue json)

blockUpdatedInfoFromJson

public static BlockUpdatedInfo blockUpdatedInfoFromJson(org.json4s.JsonAST.JValue json)

resourcesMapFromJson

public static scala.collection.Map<String,ResourceInformation> resourcesMapFromJson(org.json4s.JsonAST.JValue json)

mapFromJson

public static scala.collection.Map<String,String> mapFromJson(org.json4s.JsonAST.JValue json)

-------------------------------- * Util JSON deserialization methods |

Parameters:

json - (undocumented)

Returns:

(undocumented) ---------------------------------

propertiesFromJson

public static java.util.Properties propertiesFromJson(org.json4s.JsonAST.JValue json)

UUIDFromJson

public static java.util.UUID UUIDFromJson(org.json4s.JsonAST.JValue json)

stackTraceFromJson

public static StackTraceElement[] stackTraceFromJson(org.json4s.JsonAST.JValue json)

exceptionFromJson

public static Exception exceptionFromJson(org.json4s.JsonAST.JValue json)