Class: SparkConnect::SparkSession::Builder

Inherits:
Object
  • Object
show all
Defined in:
lib/spark_connect/session.rb

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Builder

Returns a new instance of Builder.



267
268
269
270
 
# File 'lib/spark_connect/session.rb', line 267

def initialize
  @options = {}
  @remote = nil
end

Instance Method Details

#app_name(name) ⇒ self

Set the application name.

Returns:

  • (self) 


281
282
283
284
 
# File 'lib/spark_connect/session.rb', line 281

def app_name(name)
  @options["spark.app.name"] = name
  self
end

#config(key, value) ⇒ self

Set an arbitrary configuration option to apply after connecting.

Returns:

  • (self) 


288
289
290
291
 
# File 'lib/spark_connect/session.rb', line 288

def config(key, value)
  @options[key.to_s] = value
  self
end

#create ⇒ SparkSession Also known as: build

Always build a new SparkConnect::SparkSession.

Returns:



307
308
309
310
311
312
313
 
# File 'lib/spark_connect/session.rb', line 307

def create
  url = @remote || ENV["SPARK_REMOTE"] || "sc://localhost:15002"
  client = SparkConnectClient.new(ChannelBuilder.new(url))
  session = SparkSession.new(client)
  @options.each { |k, v| session.conf.set(k, v) unless k == "spark.app.name" }
  session
end

#get_or_create ⇒ SparkSession Also known as: getOrCreate

Build (or reuse the active) SparkConnect::SparkSession.

Returns:



295
296
297
298
299
300
301
302
 
# File 'lib/spark_connect/session.rb', line 295

def get_or_create
  existing = SparkSession.active
  return existing if existing

  session = create
  SparkSession.active = session
  session
end

#remote(url) ⇒ self

Set the connection string (sc://...).

Returns:

  • (self) 


274
275
276
277
 
# File 'lib/spark_connect/session.rb', line 274

def remote(url)
  @remote = url
  self
end