functions

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
def abs(e: Column): Column

Computes the absolute value.
Computes the absolute value.

Since
1.3.0
def acos(columnName: String): Column

Computes the cosine inverse of the given column; the returned angle is in the range 0.
Computes the cosine inverse of the given column; the returned angle is in the range 0.0 through pi.

Since
1.4.0
def acos(e: Column): Column

Computes the cosine inverse of the given value; the returned angle is in the range 0.
Computes the cosine inverse of the given value; the returned angle is in the range 0.0 through pi.

Since
1.4.0
def approxCountDistinct(columnName: String, rsd: Double): Column

Aggregate function: returns the approximate number of distinct items in a group.
Aggregate function: returns the approximate number of distinct items in a group.

Since
1.3.0
def approxCountDistinct(e: Column, rsd: Double): Column

Aggregate function: returns the approximate number of distinct items in a group.
Aggregate function: returns the approximate number of distinct items in a group.

Since
1.3.0
def approxCountDistinct(columnName: String): Column

Aggregate function: returns the approximate number of distinct items in a group.
Aggregate function: returns the approximate number of distinct items in a group.

Since
1.3.0
def approxCountDistinct(e: Column): Column

Aggregate function: returns the approximate number of distinct items in a group.
Aggregate function: returns the approximate number of distinct items in a group.

Since
1.3.0
def array(colName: String, colNames: String*): Column

Creates a new array column.
Creates a new array column. The input columns must all have the same data type.

Since
1.4.0
def array(cols: Column*): Column

Creates a new array column.
Creates a new array column. The input columns must all have the same data type.

Annotations
@varargs()
Since
1.4.0
final def asInstanceOf[T0]: T0

Definition Classes
Any
def asc(columnName: String): Column

Returns a sort expression based on ascending order of the column.
Returns a sort expression based on ascending order of the column.
```
// Sort by dept in ascending order, and then age in descending order.
df.sort(asc("dept"), desc("age"))
```
Since
1.3.0
def asin(columnName: String): Column

Computes the sine inverse of the given column; the returned angle is in the range -pi/2 through pi/2.
Computes the sine inverse of the given column; the returned angle is in the range -pi/2 through pi/2.

Since
1.4.0
def asin(e: Column): Column

Computes the sine inverse of the given value; the returned angle is in the range -pi/2 through pi/2.
Computes the sine inverse of the given value; the returned angle is in the range -pi/2 through pi/2.

Since
1.4.0
def atan(columnName: String): Column

Computes the tangent inverse of the given column.
Computes the tangent inverse of the given column.

Since
1.4.0
def atan(e: Column): Column

Computes the tangent inverse of the given value.
Computes the tangent inverse of the given value.

Since
1.4.0
def atan2(l: Double, rightName: String): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(l: Double, r: Column): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(leftName: String, r: Double): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(l: Column, r: Double): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(leftName: String, rightName: String): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(leftName: String, r: Column): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(l: Column, rightName: String): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def atan2(l: Column, r: Column): Column

Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).
Returns the angle theta from the conversion of rectangular coordinates (x, y) to polar coordinates (r, theta).

Since
1.4.0
def avg(columnName: String): Column

Aggregate function: returns the average of the values in a group.
Aggregate function: returns the average of the values in a group.

Since
1.3.0
def avg(e: Column): Column

Aggregate function: returns the average of the values in a group.
Aggregate function: returns the average of the values in a group.

Since
1.3.0
def bitwiseNOT(e: Column): Column

Computes bitwise NOT.
Computes bitwise NOT.

Since
1.4.0
def callUDF(f: Function10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column): Column

Call a Scala function of 10 arguments as user-defined function (UDF).
Call a Scala function of 10 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function9[_, _, _, _, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column): Column

Call a Scala function of 9 arguments as user-defined function (UDF).
Call a Scala function of 9 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function8[_, _, _, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column): Column

Call a Scala function of 8 arguments as user-defined function (UDF).
Call a Scala function of 8 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function7[_, _, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column): Column

Call a Scala function of 7 arguments as user-defined function (UDF).
Call a Scala function of 7 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function6[_, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column): Column

Call a Scala function of 6 arguments as user-defined function (UDF).
Call a Scala function of 6 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function5[_, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column): Column

Call a Scala function of 5 arguments as user-defined function (UDF).
Call a Scala function of 5 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function4[_, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column): Column

Call a Scala function of 4 arguments as user-defined function (UDF).
Call a Scala function of 4 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function3[_, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column): Column

Call a Scala function of 3 arguments as user-defined function (UDF).
Call a Scala function of 3 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function2[_, _, _], returnType: DataType, arg1: Column, arg2: Column): Column

Call a Scala function of 2 arguments as user-defined function (UDF).
Call a Scala function of 2 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function1[_, _], returnType: DataType, arg1: Column): Column

Call a Scala function of 1 arguments as user-defined function (UDF).
Call a Scala function of 1 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0
def callUDF(f: Function0[_], returnType: DataType): Column

Call a Scala function of 0 arguments as user-defined function (UDF).
Call a Scala function of 0 arguments as user-defined function (UDF). This requires you to specify the return data type.

Since
1.3.0

def callUdf(udfName: String, cols: Column*): Column

Call an user-defined function.

Call an user-defined function. Example:

import org.apache.spark.sql._

val df = Seq(("id1", 1), ("id2", 4), ("id3", 5)).toDF("id", "value")
val sqlContext = df.sqlContext
sqlContext.udf.register("simpleUdf", (v: Int) => v * v)
df.select($"id", callUdf("simpleUdf", $"value"))

Since: 1.4.0

def cbrt(columnName: String): Column

Computes the cube-root of the given column.
Computes the cube-root of the given column.

Since
1.4.0
def cbrt(e: Column): Column

Computes the cube-root of the given value.
Computes the cube-root of the given value.

Since
1.4.0
def ceil(columnName: String): Column

Computes the ceiling of the given column.
Computes the ceiling of the given column.

Since
1.4.0
def ceil(e: Column): Column

Computes the ceiling of the given value.
Computes the ceiling of the given value.

Since
1.4.0
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def coalesce(e: Column*): Column

Returns the first column that is not null.
Returns the first column that is not null.
```
df.select(coalesce(df("a"), df("b")))
```
Annotations
@varargs()
Since
1.3.0
def col(colName: String): Column

Returns a Column based on the given column name.
Returns a Column based on the given column name.

Since
1.3.0
def column(colName: String): Column

Returns a Column based on the given column name.
Returns a Column based on the given column name. Alias of col.

Since
1.3.0
def cos(columnName: String): Column

Computes the cosine of the given column.
Computes the cosine of the given column.

Since
1.4.0
def cos(e: Column): Column

Computes the cosine of the given value.
Computes the cosine of the given value.

Since
1.4.0
def cosh(columnName: String): Column

Computes the hyperbolic cosine of the given column.
Computes the hyperbolic cosine of the given column.

Since
1.4.0
def cosh(e: Column): Column

Computes the hyperbolic cosine of the given value.
Computes the hyperbolic cosine of the given value.

Since
1.4.0
def count(columnName: String): Column

Aggregate function: returns the number of items in a group.
Aggregate function: returns the number of items in a group.

Since
1.3.0
def count(e: Column): Column

Aggregate function: returns the number of items in a group.
Aggregate function: returns the number of items in a group.

Since
1.3.0
def countDistinct(columnName: String, columnNames: String*): Column

Aggregate function: returns the number of distinct items in a group.
Aggregate function: returns the number of distinct items in a group.

Annotations
@varargs()
Since
1.3.0
def countDistinct(expr: Column, exprs: Column*): Column

Aggregate function: returns the number of distinct items in a group.
Aggregate function: returns the number of distinct items in a group.

Annotations
@varargs()
Since
1.3.0
def cumeDist(): Column

Window function: returns the cumulative distribution of values within a window partition, i.
Window function: returns the cumulative distribution of values within a window partition, i.e. the fraction of rows that are below the current row.
```
N = total number of rows in the partition
cumeDist(x) = number of values before (and including) x / N
```
This is equivalent to the CUME_DIST function in SQL.
Since
1.4.0
def denseRank(): Column

Window function: returns the rank of rows within a window partition, without any gaps.
Window function: returns the rank of rows within a window partition, without any gaps.
The difference between rank and denseRank is that denseRank leaves no gaps in ranking sequence when there are ties. That is, if you were ranking a competition using denseRank and had three people tie for second place, you would say that all three were in second place and that the next person came in third.
This is equivalent to the DENSE_RANK function in SQL.

Since
1.4.0
def desc(columnName: String): Column

Returns a sort expression based on the descending order of the column.
Returns a sort expression based on the descending order of the column.
```
// Sort by dept in ascending order, and then age in descending order.
df.sort(asc("dept"), desc("age"))
```
Since
1.3.0
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def exp(columnName: String): Column

Computes the exponential of the given column.
Computes the exponential of the given column.

Since
1.4.0
def exp(e: Column): Column

Computes the exponential of the given value.
Computes the exponential of the given value.

Since
1.4.0
def explode(e: Column): Column

Creates a new row for each element in the given array or map column.
def expm1(columnName: String): Column

Computes the exponential of the given column.
Computes the exponential of the given column.

Since
1.4.0
def expm1(e: Column): Column

Computes the exponential of the given value minus one.
Computes the exponential of the given value minus one.

Since
1.4.0
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def first(columnName: String): Column

Aggregate function: returns the first value of a column in a group.
Aggregate function: returns the first value of a column in a group.

Since
1.3.0
def first(e: Column): Column

Aggregate function: returns the first value in a group.
Aggregate function: returns the first value in a group.

Since
1.3.0
def floor(columnName: String): Column

Computes the floor of the given column.
Computes the floor of the given column.

Since
1.4.0
def floor(e: Column): Column

Computes the floor of the given value.
Computes the floor of the given value.

Since
1.4.0
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def hypot(l: Double, rightName: String): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(l: Double, r: Column): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(leftName: String, r: Double): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(l: Column, r: Double): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(leftName: String, rightName: String): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(leftName: String, r: Column): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(l: Column, rightName: String): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
def hypot(l: Column, r: Column): Column

Computes sqrt(a² + b²) without intermediate overflow or underflow.
Computes sqrt(a² + b²) without intermediate overflow or underflow.

Since
1.4.0
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def lag(e: Column, offset: Int, defaultValue: Any): Column

Window function: returns the value that is offset rows before the current row, and defaultValue if there is less than offset rows before the current row.
Window function: returns the value that is offset rows before the current row, and defaultValue if there is less than offset rows before the current row. For example, an offset of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.

Since
1.4.0
def lag(columnName: String, offset: Int, defaultValue: Any): Column

Window function: returns the value that is offset rows before the current row, and defaultValue if there is less than offset rows before the current row.
Window function: returns the value that is offset rows before the current row, and defaultValue if there is less than offset rows before the current row. For example, an offset of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.

Since
1.4.0
def lag(columnName: String, offset: Int): Column

Window function: returns the value that is offset rows before the current row, and null if there is less than offset rows before the current row.
Window function: returns the value that is offset rows before the current row, and null if there is less than offset rows before the current row. For example, an offset of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.

Since
1.4.0
def lag(e: Column, offset: Int): Column

Window function: returns the value that is offset rows before the current row, and null if there is less than offset rows before the current row.
Window function: returns the value that is offset rows before the current row, and null if there is less than offset rows before the current row. For example, an offset of one will return the previous row at any given point in the window partition.
This is equivalent to the LAG function in SQL.

Since
1.4.0
def last(columnName: String): Column

Aggregate function: returns the last value of the column in a group.
Aggregate function: returns the last value of the column in a group.

Since
1.3.0
def last(e: Column): Column

Aggregate function: returns the last value in a group.
Aggregate function: returns the last value in a group.

Since
1.3.0
def lead(e: Column, offset: Int, defaultValue: Any): Column

Window function: returns the value that is offset rows after the current row, and defaultValue if there is less than offset rows after the current row.
Window function: returns the value that is offset rows after the current row, and defaultValue if there is less than offset rows after the current row. For example, an offset of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.

Since
1.4.0
def lead(columnName: String, offset: Int, defaultValue: Any): Column

Window function: returns the value that is offset rows after the current row, and defaultValue if there is less than offset rows after the current row.
Window function: returns the value that is offset rows after the current row, and defaultValue if there is less than offset rows after the current row. For example, an offset of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.

Since
1.4.0
def lead(e: Column, offset: Int): Column

Window function: returns the value that is offset rows after the current row, and null if there is less than offset rows after the current row.
Window function: returns the value that is offset rows after the current row, and null if there is less than offset rows after the current row. For example, an offset of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.

Since
1.4.0
def lead(columnName: String, offset: Int): Column

Window function: returns the value that is offset rows after the current row, and null if there is less than offset rows after the current row.
Window function: returns the value that is offset rows after the current row, and null if there is less than offset rows after the current row. For example, an offset of one will return the next row at any given point in the window partition.
This is equivalent to the LEAD function in SQL.

Since
1.4.0
def lit(literal: Any): Column

Creates a Column of literal value.
Creates a Column of literal value.
The passed in object is returned directly if it is already a Column. If the object is a Scala Symbol, it is converted into a Column also. Otherwise, a new Column is created to represent the literal value.

Since
1.3.0
def log(columnName: String): Column

Computes the natural logarithm of the given column.
Computes the natural logarithm of the given column.

Since
1.4.0
def log(e: Column): Column

Computes the natural logarithm of the given value.
Computes the natural logarithm of the given value.

Since
1.4.0
def log10(columnName: String): Column

Computes the logarithm of the given value in Base 10.
Computes the logarithm of the given value in Base 10.

Since
1.4.0
def log10(e: Column): Column

Computes the logarithm of the given value in Base 10.
Computes the logarithm of the given value in Base 10.

Since
1.4.0
def log1p(columnName: String): Column

Computes the natural logarithm of the given column plus one.
Computes the natural logarithm of the given column plus one.

Since
1.4.0
def log1p(e: Column): Column

Computes the natural logarithm of the given value plus one.
Computes the natural logarithm of the given value plus one.

Since
1.4.0
def lower(e: Column): Column

Converts a string exprsesion to lower case.
Converts a string exprsesion to lower case.

Since
1.3.0
def max(columnName: String): Column

Aggregate function: returns the maximum value of the column in a group.
Aggregate function: returns the maximum value of the column in a group.

Since
1.3.0
def max(e: Column): Column

Aggregate function: returns the maximum value of the expression in a group.
Aggregate function: returns the maximum value of the expression in a group.

Since
1.3.0
def mean(columnName: String): Column

Aggregate function: returns the average of the values in a group.
Aggregate function: returns the average of the values in a group. Alias for avg.

Since
1.4.0
def mean(e: Column): Column

Aggregate function: returns the average of the values in a group.
Aggregate function: returns the average of the values in a group. Alias for avg.

Since
1.4.0
def min(columnName: String): Column

Aggregate function: returns the minimum value of the column in a group.
Aggregate function: returns the minimum value of the column in a group.

Since
1.3.0
def min(e: Column): Column

Aggregate function: returns the minimum value of the expression in a group.
Aggregate function: returns the minimum value of the expression in a group.

Since
1.3.0
def monotonicallyIncreasingId(): Column

A column expression that generates monotonically increasing 64-bit integers.
A column expression that generates monotonically increasing 64-bit integers.
The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. The current implementation puts the partition ID in the upper 31 bits, and the record number within each partition in the lower 33 bits. The assumption is that the data frame has less than 1 billion partitions, and each partition has less than 8 billion records.
As an example, consider a DataFrame with two partitions, each with 3 records. This expression would return the following IDs: 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594.

Since
1.4.0
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef

def negate(e: Column): Column

Unary minus, i.

Unary minus, i.e. negate the expression.

// Select the amount column and negates all values.
// Scala:
df.select( -df("amount") )

// Java:
df.select( negate(df.col("amount")) );

Since: 1.3.0

def not(e: Column): Column

Inversion of boolean expression, i.

Inversion of boolean expression, i.e. NOT.

// Scala: select rows that are not active (isActive === false)
df.filter( !df("isActive") )

// Java:
df.filter( not(df.col("isActive")) );

Since: 1.3.0

final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def ntile(n: Int): Column

Window function: returns the ntile group id (from 1 to n inclusive) in an ordered window partition.
Window function: returns the ntile group id (from 1 to n inclusive) in an ordered window partition. Fow example, if n is 4, the first quarter of the rows will get value 1, the second quarter will get 2, the third quarter will get 3, and the last quarter will get 4.
This is equivalent to the NTILE function in SQL.

Since
1.4.0
def percentRank(): Column

Window function: returns the relative rank (i.
Window function: returns the relative rank (i.e. percentile) of rows within a window partition.
This is computed by:
```
(rank of row in its partition - 1) / (number of rows in the partition - 1)
```
This is equivalent to the PERCENT_RANK function in SQL.
Since
1.4.0
def pow(l: Double, rightName: String): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(l: Double, r: Column): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(leftName: String, r: Double): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(l: Column, r: Double): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(leftName: String, rightName: String): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(leftName: String, r: Column): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(l: Column, rightName: String): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def pow(l: Column, r: Column): Column

Returns the value of the first argument raised to the power of the second argument.
Returns the value of the first argument raised to the power of the second argument.

Since
1.4.0
def rand(): Column

Generate a random column with i.
Generate a random column with i.i.d. samples from U[0.0, 1.0].

Since
1.4.0
def rand(seed: Long): Column

Generate a random column with i.
Generate a random column with i.i.d. samples from U[0.0, 1.0].

Since
1.4.0
def randn(): Column

Generate a column with i.
Generate a column with i.i.d. samples from the standard normal distribution.

Since
1.4.0
def randn(seed: Long): Column

Generate a column with i.
Generate a column with i.i.d. samples from the standard normal distribution.

Since
1.4.0
def rank(): Column

Window function: returns the rank of rows within a window partition.
Window function: returns the rank of rows within a window partition.
The difference between rank and denseRank is that denseRank leaves no gaps in ranking sequence when there are ties. That is, if you were ranking a competition using denseRank and had three people tie for second place, you would say that all three were in second place and that the next person came in third.
This is equivalent to the RANK function in SQL.

Since
1.4.0
def rint(columnName: String): Column

Returns the double value that is closest in value to the argument and is equal to a mathematical integer.
Returns the double value that is closest in value to the argument and is equal to a mathematical integer.

Since
1.4.0
def rint(e: Column): Column

Returns the double value that is closest in value to the argument and is equal to a mathematical integer.
Returns the double value that is closest in value to the argument and is equal to a mathematical integer.

Since
1.4.0
def rowNumber(): Column

Window function: returns a sequential number starting at 1 within a window partition.
Window function: returns a sequential number starting at 1 within a window partition.
This is equivalent to the ROW_NUMBER function in SQL.

Since
1.4.0
def signum(columnName: String): Column

Computes the signum of the given column.
Computes the signum of the given column.

Since
1.4.0
def signum(e: Column): Column

Computes the signum of the given value.
Computes the signum of the given value.

Since
1.4.0
def sin(columnName: String): Column

Computes the sine of the given column.
Computes the sine of the given column.

Since
1.4.0
def sin(e: Column): Column

Computes the sine of the given value.
Computes the sine of the given value.

Since
1.4.0
def sinh(columnName: String): Column

Computes the hyperbolic sine of the given column.
Computes the hyperbolic sine of the given column.

Since
1.4.0
def sinh(e: Column): Column

Computes the hyperbolic sine of the given value.
Computes the hyperbolic sine of the given value.

Since
1.4.0
def sparkPartitionId(): Column

Partition ID of the Spark task.
Partition ID of the Spark task.
Note that this is indeterministic because it depends on data partitioning and task scheduling.

Since
1.4.0
def sqrt(e: Column): Column

Computes the square root of the specified float value.
Computes the square root of the specified float value.

Since
1.3.0
def struct(colName: String, colNames: String*): Column

Creates a new struct column that composes multiple input columns.
Creates a new struct column that composes multiple input columns.

Since
1.4.0
def struct(cols: Column*): Column

Creates a new struct column.
Creates a new struct column. The input column must be a column in a DataFrame, or a derived column expression that is named (i.e. aliased).

Annotations
@varargs()
Since
1.4.0
def sum(columnName: String): Column

Aggregate function: returns the sum of all values in the given column.
Aggregate function: returns the sum of all values in the given column.

Since
1.3.0
def sum(e: Column): Column

Aggregate function: returns the sum of all values in the expression.
Aggregate function: returns the sum of all values in the expression.

Since
1.3.0
def sumDistinct(columnName: String): Column

Aggregate function: returns the sum of distinct values in the expression.
Aggregate function: returns the sum of distinct values in the expression.

Since
1.3.0
def sumDistinct(e: Column): Column

Aggregate function: returns the sum of distinct values in the expression.
Aggregate function: returns the sum of distinct values in the expression.

Since
1.3.0
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def tan(columnName: String): Column

Computes the tangent of the given column.
Computes the tangent of the given column.

Since
1.4.0
def tan(e: Column): Column

Computes the tangent of the given value.
Computes the tangent of the given value.

Since
1.4.0
def tanh(columnName: String): Column

Computes the hyperbolic tangent of the given column.
Computes the hyperbolic tangent of the given column.

Since
1.4.0
def tanh(e: Column): Column

Computes the hyperbolic tangent of the given value.
Computes the hyperbolic tangent of the given value.

Since
1.4.0
def toDegrees(columnName: String): Column

Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
Converts an angle measured in radians to an approximately equivalent angle measured in degrees.

Since
1.4.0
def toDegrees(e: Column): Column

Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
Converts an angle measured in radians to an approximately equivalent angle measured in degrees.

Since
1.4.0
def toRadians(columnName: String): Column

Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
Converts an angle measured in degrees to an approximately equivalent angle measured in radians.

Since
1.4.0
def toRadians(e: Column): Column

Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
Converts an angle measured in degrees to an approximately equivalent angle measured in radians.

Since
1.4.0
def toString(): String

Definition Classes
AnyRef → Any
def udf[RT, A1, A2, A3, A4, A5, A6, A7, A8, A9, A10](f: (A1, A2, A3, A4, A5, A6, A7, A8, A9, A10) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6], arg7: scala.reflect.api.JavaUniverse.TypeTag[A7], arg8: scala.reflect.api.JavaUniverse.TypeTag[A8], arg9: scala.reflect.api.JavaUniverse.TypeTag[A9], arg10: scala.reflect.api.JavaUniverse.TypeTag[A10]): UserDefinedFunction

Defines a user-defined function of 10 arguments as user-defined function (UDF).
Defines a user-defined function of 10 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5, A6, A7, A8, A9](f: (A1, A2, A3, A4, A5, A6, A7, A8, A9) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6], arg7: scala.reflect.api.JavaUniverse.TypeTag[A7], arg8: scala.reflect.api.JavaUniverse.TypeTag[A8], arg9: scala.reflect.api.JavaUniverse.TypeTag[A9]): UserDefinedFunction

Defines a user-defined function of 9 arguments as user-defined function (UDF).
Defines a user-defined function of 9 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5, A6, A7, A8](f: (A1, A2, A3, A4, A5, A6, A7, A8) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6], arg7: scala.reflect.api.JavaUniverse.TypeTag[A7], arg8: scala.reflect.api.JavaUniverse.TypeTag[A8]): UserDefinedFunction

Defines a user-defined function of 8 arguments as user-defined function (UDF).
Defines a user-defined function of 8 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5, A6, A7](f: (A1, A2, A3, A4, A5, A6, A7) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6], arg7: scala.reflect.api.JavaUniverse.TypeTag[A7]): UserDefinedFunction

Defines a user-defined function of 7 arguments as user-defined function (UDF).
Defines a user-defined function of 7 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5, A6](f: (A1, A2, A3, A4, A5, A6) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5], arg6: scala.reflect.api.JavaUniverse.TypeTag[A6]): UserDefinedFunction

Defines a user-defined function of 6 arguments as user-defined function (UDF).
Defines a user-defined function of 6 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1, A2, A3, A4, A5](f: (A1, A2, A3, A4, A5) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4], arg5: scala.reflect.api.JavaUniverse.TypeTag[A5]): UserDefinedFunction

Defines a user-defined function of 5 arguments as user-defined function (UDF).
Defines a user-defined function of 5 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1, A2, A3, A4](f: (A1, A2, A3, A4) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3], arg4: scala.reflect.api.JavaUniverse.TypeTag[A4]): UserDefinedFunction

Defines a user-defined function of 4 arguments as user-defined function (UDF).
Defines a user-defined function of 4 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1, A2, A3](f: (A1, A2, A3) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2], arg3: scala.reflect.api.JavaUniverse.TypeTag[A3]): UserDefinedFunction

Defines a user-defined function of 3 arguments as user-defined function (UDF).
Defines a user-defined function of 3 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1, A2](f: (A1, A2) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1], arg2: scala.reflect.api.JavaUniverse.TypeTag[A2]): UserDefinedFunction

Defines a user-defined function of 2 arguments as user-defined function (UDF).
Defines a user-defined function of 2 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT, A1](f: (A1) ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT], arg1: scala.reflect.api.JavaUniverse.TypeTag[A1]): UserDefinedFunction

Defines a user-defined function of 1 arguments as user-defined function (UDF).
Defines a user-defined function of 1 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def udf[RT](f: () ⇒ RT)(implicit arg0: scala.reflect.api.JavaUniverse.TypeTag[RT]): UserDefinedFunction

Defines a user-defined function of 0 arguments as user-defined function (UDF).
Defines a user-defined function of 0 arguments as user-defined function (UDF). The data types are automatically inferred based on the function's signature.

Since
1.3.0
def upper(e: Column): Column

Converts a string expression to upper case.
Converts a string expression to upper case.

Since
1.3.0
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

def when(condition: Column, value: Any): Column

Evaluates a list of conditions and returns one of multiple possible result expressions.

Evaluates a list of conditions and returns one of multiple possible result expressions. If otherwise is not defined at the end, null is returned for unmatched conditions.

// Example: encoding gender string column into integer.

// Scala:
people.select(when(people("gender") === "male", 0)
  .when(people("gender") === "female", 1)
  .otherwise(2))

// Java:
people.select(when(col("gender").equalTo("male"), 0)
  .when(col("gender").equalTo("female"), 1)
  .otherwise(2))

object functions

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

def abs(e: Column): Column

def acos(columnName: String): Column

def acos(e: Column): Column

def approxCountDistinct(columnName: String, rsd: Double): Column

def approxCountDistinct(e: Column, rsd: Double): Column

def approxCountDistinct(columnName: String): Column

def approxCountDistinct(e: Column): Column

def array(colName: String, colNames: String*): Column

def array(cols: Column*): Column

final def asInstanceOf[T0]: T0

def asc(columnName: String): Column

def asin(columnName: String): Column

def asin(e: Column): Column

def atan(columnName: String): Column

def atan(e: Column): Column

def atan2(l: Double, rightName: String): Column

def atan2(l: Double, r: Column): Column

def atan2(leftName: String, r: Double): Column

def atan2(l: Column, r: Double): Column

def atan2(leftName: String, rightName: String): Column

def atan2(leftName: String, r: Column): Column

def atan2(l: Column, rightName: String): Column

def atan2(l: Column, r: Column): Column

def avg(columnName: String): Column

def avg(e: Column): Column

def bitwiseNOT(e: Column): Column

def callUDF(f: Function10[_, _, _, _, _, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column, arg10: Column): Column

def callUDF(f: Function9[_, _, _, _, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column, arg9: Column): Column

def callUDF(f: Function8[_, _, _, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column, arg8: Column): Column

def callUDF(f: Function7[_, _, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column, arg7: Column): Column

def callUDF(f: Function6[_, _, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column, arg6: Column): Column

def callUDF(f: Function5[_, _, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column, arg5: Column): Column

def callUDF(f: Function4[_, _, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column, arg4: Column): Column

def callUDF(f: Function3[_, _, _, _], returnType: DataType, arg1: Column, arg2: Column, arg3: Column): Column

def callUDF(f: Function2[_, _, _], returnType: DataType, arg1: Column, arg2: Column): Column

def callUDF(f: Function1[_, _], returnType: DataType, arg1: Column): Column

def callUDF(f: Function0[_], returnType: DataType): Column

def callUdf(udfName: String, cols: Column*): Column

def cbrt(columnName: String): Column

def cbrt(e: Column): Column

def ceil(columnName: String): Column

def ceil(e: Column): Column

def clone(): AnyRef

def coalesce(e: Column*): Column

def col(colName: String): Column

def column(colName: String): Column

def cos(columnName: String): Column

def cos(e: Column): Column

def cosh(columnName: String): Column

def cosh(e: Column): Column

def count(columnName: String): Column

def count(e: Column): Column

def countDistinct(columnName: String, columnNames: String*): Column

def countDistinct(expr: Column, exprs: Column*): Column

def cumeDist(): Column

def denseRank(): Column

def desc(columnName: String): Column

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def exp(columnName: String): Column

def exp(e: Column): Column

def explode(e: Column): Column

def expm1(columnName: String): Column

def expm1(e: Column): Column

def finalize(): Unit

def first(columnName: String): Column

def first(e: Column): Column

def floor(columnName: String): Column

def floor(e: Column): Column

final def getClass(): Class[_]

def hashCode(): Int

def hypot(l: Double, rightName: String): Column