[SparkR-239] buildSchema and field functions
#235
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,145 @@ | ||
| # A set of S3 classes and methods that support the SparkSQL `StructType` and `StructField | ||
| # datatypes. These are used to create and interact with DataFrame schemas. | ||
|
|
||
| #' structType | ||
|
There was a problem hiding this comment. A one line comment on top with a description of what is in this file would be good There was a problem hiding this comment. Good call. Fixing that now. |
||
| #' | ||
| #' Create a structType object that contains the metadata for a DataFrame. Intended for | ||
| #' use with createDataFrame and toDF. | ||
| #' | ||
| #' @param x a structField object (created with the field() function) | ||
| #' @param ... additional structField objects | ||
| #' @return a structType object | ||
| #' @export | ||
| #' @examples | ||
| #'\dontrun{ | ||
| #' sc <- sparkR.init() | ||
| #' sqlCtx <- sparkRSQL.init(sc) | ||
| #' rdd <- lapply(parallelize(sc, 1:10), function(x) { list(x, as.character(x)) }) | ||
| #' schema <- structType(structField("a", "integer"), structField("b", "string")) | ||
| #' df <- createDataFrame(sqlCtx, rdd, schema) | ||
| #' } | ||
| structType <- function(x, ...) { | ||
| UseMethod("structType", x) | ||
| } | ||
|
|
||
| structType.jobj <- function(x) { | ||
| obj <- structure(list(), class = "structType") | ||
| obj$jobj <- x | ||
| obj$fields <- function() { lapply(callJMethod(obj$jobj, "fields"), structField) } | ||
| obj | ||
| } | ||
|
|
||
| structType.structField <- function(x, ...) { | ||
| fields <- list(x, ...) | ||
| if (!all(sapply(fields, inherits, "structField"))) { | ||
| stop("All arguments must be structField objects.") | ||
| } | ||
| sfObjList <- lapply(fields, function(field) { | ||
| field$jobj | ||
| }) | ||
| stObj <- callJStatic("edu.berkeley.cs.amplab.sparkr.SQLUtils", | ||
| "createStructType", | ||
| listToSeq(sfObjList)) | ||
| structType(stObj) | ||
| } | ||
|
|
||
| #' Print a Spark StructType. | ||
| #' | ||
| #' This function prints the contents of a StructType returned from the | ||
| #' SparkR JVM backend. | ||
| #' | ||
| #' @param x A StructType object | ||
| #' @param ... further arguments passed to or from other methods | ||
| print.structType <- function(x, ...) { | ||
| cat("StructType\n", | ||
| sapply(x$fields(), function(field) { paste("|-", "name = \"", field$name(), | ||
| "\", type = \"", field$dataType.toString(), | ||
| "\", nullable = ", field$nullable(), "\n", | ||
| sep = "") }) | ||
| , sep = "") | ||
| } | ||
|
|
||
| #' structField | ||
| #' | ||
| #' Create a structField object that contains the metadata for a single field in a schema. | ||
| #' | ||
| #' @param x The name of the field | ||
| #' @param type The data type of the field | ||
| #' @param nullable A logical vector indicating whether or not the field is nullable | ||
| #' @return a structField object | ||
| #' @export | ||
| #' @examples | ||
| #'\dontrun{ | ||
| #' sc <- sparkR.init() | ||
| #' sqlCtx <- sparkRSQL.init(sc) | ||
| #' rdd <- lapply(parallelize(sc, 1:10), function(x) { list(x, as.character(x)) }) | ||
| #' field1 <- structField("a", "integer", TRUE) | ||
| #' field2 <- structField("b", "string", TRUE) | ||
| #' schema <- structType(field1, field2) | ||
| #' df <- createDataFrame(sqlCtx, rdd, schema) | ||
| #' } | ||
|
|
||
| structField <- function(x, ...) { | ||
| UseMethod("structField", x) | ||
| } | ||
|
|
||
| structField.jobj <- function(x) { | ||
| obj <- structure(list(), class = "structField") | ||
| obj$jobj <- x | ||
| obj$name <- function() { callJMethod(x, "name") } | ||
| obj$dataType <- function() { callJMethod(x, "dataType") } | ||
| obj$dataType.toString <- function() { callJMethod(obj$dataType(), "toString") } | ||
| obj$dataType.simpleString <- function() { callJMethod(obj$dataType(), "simpleString") } | ||
| obj$nullable <- function() { callJMethod(x, "nullable") } | ||
| obj | ||
| } | ||
|
|
||
| structField.character <- function(x, type, nullable = TRUE) { | ||
| if (class(x) != "character") { | ||
| stop("Field name must be a string.") | ||
| } | ||
| if (class(type) != "character") { | ||
| stop("Field type must be a string.") | ||
| } | ||
| if (class(nullable) != "logical") { | ||
| stop("nullable must be either TRUE or FALSE") | ||
| } | ||
| options <- c("byte", | ||
| "integer", | ||
| "double", | ||
| "numeric", | ||
| "character", | ||
| "string", | ||
| "binary", | ||
| "raw", | ||
| "logical", | ||
| "boolean", | ||
| "timestamp", | ||
| "date") | ||
| dataType <- if (type %in% options) { | ||
| type | ||
| } else { | ||
| stop(paste("Unsupported type for Dataframe:", type)) | ||
| } | ||
| sfObj <- callJStatic("edu.berkeley.cs.amplab.sparkr.SQLUtils", | ||
| "createStructField", | ||
| x, | ||
| dataType, | ||
| nullable) | ||
| structField(sfObj) | ||
| } | ||
|
|
||
| #' Print a Spark StructField. | ||
| #' | ||
| #' This function prints the contents of a StructField returned from the | ||
| #' SparkR JVM backend. | ||
| #' | ||
| #' @param x A StructField object | ||
| #' @param ... further arguments passed to or from other methods | ||
| print.structField <- function(x, ...) { | ||
| cat("StructField(name = \"", x$name(), | ||
| "\", type = \"", x$dataType.toString(), | ||
| "\", nullable = ", x$nullable(), | ||
| ")", | ||
| sep = "") | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just curious: does the
structFieldconstructor fail if we don't exportstructField.jobj?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It does in some cases. For example, when you call the
fields()closure of a pre-existingstructTypeobject, at that point you're trying to explicitly create astructFieldfrom ajobjdirectly and it won't be able to find the correct method: