Class: SparkConnect::RuntimeConfig

Inherits:
Object
  • Object
show all
Defined in:
lib/spark_connect/conf.rb

Overview

Runtime configuration interface, returned by SparkSession#conf. Mirrors PySpark's spark.conf.

Examples:

spark.conf.set("spark.sql.shuffle.partitions", "8")
spark.conf.get("spark.sql.shuffle.partitions")  #=> "8"

Constant Summary collapse

Proto = 
SparkConnect::Proto
Op = 
Proto::ConfigRequest::Operation
CR = 
Proto::ConfigRequest

Instance Method Summary collapse

Constructor Details

#initialize(client) ⇒ RuntimeConfig

Returns a new instance of RuntimeConfig.

Parameters:



16
17
18
 
# File 'lib/spark_connect/conf.rb', line 16

def initialize(client)
  @client = client
end

Instance Method Details

#get(key, default = :__unset__) ⇒ String?

Get the value of a configuration property.

Parameters:

  • key (String)
  • default (String, nil) (defaults to: :__unset__) —

    returned when the key is unset (when given).

Returns:

  • (String, nil) 


36
37
38
39
40
41
42
43
44
45
46
47
48
 
# File 'lib/spark_connect/conf.rb', line 36

def get(key, default = :__unset__)
  op =
    if default == :__unset__
      Op.new(get: CR::Get.new(keys: [key.to_s]))
    else
      Op.new(get_with_default: CR::GetWithDefault.new(
        pairs: [Proto::KeyValue.new(key: key.to_s, value: default)]
      ))
    end
  resp = @client.config(op)
  pair = resp.pairs.first
  pair&.value
end

#get_all(prefix = nil) ⇒ Hash{String=>String}

All configuration properties (optionally filtered by prefix).

Parameters:

  • prefix (String, nil) (defaults to: nil)

Returns:

  • (Hash{String=>String}) 


63
64
65
66
67
68
 
# File 'lib/spark_connect/conf.rb', line 63

def get_all(prefix = nil)
  ga = CR::GetAll.new
  ga.prefix = prefix if prefix
  resp = @client.config(Op.new(get_all: ga))
  resp.pairs.to_h { |p| [p.key, p.value] }
end

#modifiable?(key) ⇒ Boolean

Whether a configuration property is modifiable in the current session.

Parameters:

  • key (String)

Returns:

  • (Boolean) 


74
75
76
77
 
# File 'lib/spark_connect/conf.rb', line 74

def modifiable?(key)
  resp = @client.config(Op.new(is_modifiable: CR::IsModifiable.new(keys: [key.to_s])))
  resp.pairs.first&.value == "true"
end

#set(key, value) ⇒ void

This method returns an undefined value.

Set a configuration property.

Parameters:

  • key (String)
  • value (String, Integer, Boolean) 


25
26
27
28
29
 
# File 'lib/spark_connect/conf.rb', line 25

def set(key, value)
  op = Op.new(set: CR::Set.new(pairs: [Proto::KeyValue.new(key: key.to_s, value: value.to_s)]))
  @client.config(op)
  nil
end

#unset(key) ⇒ void

This method returns an undefined value.

Unset a configuration property.

Parameters:

  • key (String) 


54
55
56
57
 
# File 'lib/spark_connect/conf.rb', line 54

def unset(key)
  @client.config(Op.new(unset: CR::Unset.new(keys: [key.to_s])))
  nil
end