pyspark中spark初始化的一些操作

spark初始化:
class SpsrkConfigInit:
@classmethod
def init_spark(cls):
“”"
初始化spark配置
:return: SparkSession
“”"
# 设置 JAVA_HOME 环境变量
os.environ[‘JAVA_HOME’] = ‘C:/Program Files/Huawei/jdk1.8.0_222’
os.environ[‘PYSPARK_PYTHON’] = ‘D:/users/xxx/AppData/Local/Programs/Python/Python310/python.exe’ #python解释器位置
# 设置日志级别
sc.setLogLevel(“WARN”)
# 创建 SparkSession
return (SparkSession.builder.appName(“SimpleApp”)
.config(“spark.master”, “local[*]”)
.config(“spark.jars”, “/tools/spark-3.5.4-bin-hadoop3/jars/mysql-connector-j-9.2.0.jar”) #连接mysql的驱动程序
.config(“spark.driver.extraClassPath”,
“/tools/spark-3.5.4-bin-hadoop3/jars/mysql-connector-j-9.2.0.jar”)
.config(“spark.debug.maxToStringFields”, “100”)
.config(“spark.local.dir”, “/users/xxx/PycharmProjects/spark/pythonProject1/temp/dir”)
.getOrCreate())