#!/bin/sh
# 任务名称
JOB_NAME=word_count
# python 文件路径
MAIN_PATH=word_count.py
# Master 节点ip
MASTER=10.202.0.207
# Spark 安装路径，无需修改，默认
SPARK_PATH=/opt/spark-3.4.1

# 环境变量，无需修改
export LD_LIBRARY_PATH=/opt/chukonu_install/lib:/opt/chukonu_cache
export LD_PRELOAD=/opt/chukonu_install/lib/libchukonu_preloaded.so:/lib/x86_64-linux-gnu/libjemalloc.so.2
CHUKONU_JAR=/opt/chukonu_install/jar/chukonu_2.12-0.5.1.jar

#任务参数，除了 spark.chukonu 开头的参数（这些参数保持不变），和spark一样，详情查看spark官方文档
DRIVER_MAX_RESULT_SIZE=1g
DRIVER_MEMORY=1g
EXECUTOR_CORES=1
EXECUTOR_MEMORY=1g
EXECUTOR_MEMORYOVERHEAD=1g
EXECUTOR_INSTANCES=1
OFFHEAP_SIZE=1g

$SPARK_PATH/bin/spark-submit \
  --deploy-mode client \
  --master spark://$MASTER:7077 \
  --name ${JOB_NAME} \
  --conf spark.kryo.unsafe=true \
  --conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
  --conf spark.executor.processTreeMetrics.enabled=true \
  --conf spark.plugins=org.pacman.chukonu.ChukonuPlugin \
  --conf spark.chukonu.enableNativeCodegen=true \
  --conf spark.chukonu.root=/opt/chukonu_install \
  --conf spark.chukonu.cxx=/usr/bin/g++ \
  --conf spark.chukonu.stagingdir=/opt/chukonu_staging \
  --conf spark.chukonu.compileCacheDir=/opt/chukonu_cache \
  --conf spark.executorEnv.LD_LIBRARY_PATH=/opt/chukonu_install/lib:/opt/chukonu_cache \
  --conf spark.executorEnv.LD_PRELOAD=/opt/chukonu_install/lib/libchukonu_preloaded.so:/lib/x86_64-linux-gnu/libjemalloc.so.2 \
  --conf spark.chukonu.buildType=Release \
  --conf spark.memory.offHeap.enabled=true \
  --conf spark.memory.offHeap.size=$OFFHEAP_SIZE \
  --conf spark.driver.memory=$DRIVER_MEMORY \
  --conf spark.driver.maxResultSize=$DRIVER_MAX_RESULT_SIZE \
  --conf spark.executor.memory=$EXECUTOR_MEMORY \
  --conf spark.executor.memoryOverhead=$EXECUTOR_MEMORYOVERHEAD \
  --conf spark.executor.cores=$EXECUTOR_CORES \
  --conf spark.executor.instances=$EXECUTOR_INSTANCES \
  --jars $CHUKONU_JAR  $MAIN_PATH