spark连接hive（spark-shell和eclipse两种方式）

cp /opt/apache-hive-2.3.2-bin/conf/hive-site.xml /opt/spark-2.2.1-bin-hadoop2.7/conf/

cp /opt/apache-hive-2.3.2-bin/bin/mysql-connector-java-5.1.46-bin.jar /opt/spark-2.2.1-bin-hadoop2.7/jars/

spark-shell
import org.apache.spark.sql.hive.HiveContext
val hiveContext = new HiveContext(sc)
hiveContext.sql("select * from test").show()

The root scratch dir: /tmp/hive on HDFS should be writable. Current permissions are: rwxr-xr-x;

hadoop fs -chmod 777 /tmp/hive

hadoop fs -rm -r /tmp/hive 
rm -rf /tmp/hive

"mysql" % "mysql-connector-java" % "5.1.46"

package com.dkl.leanring.spark.sql

import org.apache.spark.SparkConf
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.SparkContext

/**
 * 旧版本spark-hive测试
 */
object OldSparkHiveDemo {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("OldSparkHiveDemo").setMaster("local")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)
    import sqlContext.implicits._
    val hiveCtx = new HiveContext(sc)

    hiveCtx.sql("select * from test").show()
    val data = Array((3, "name3"), (4, "name4"), (5, "name5"))
    val df = sc.parallelize(data).toDF("id", "name")
    df.createOrReplaceTempView("user")
    hiveCtx.sql("insert into test select id,name from user")
    hiveCtx.sql("select * from test").show()
  }

}

package com.dkl.leanring.spark.sql

import org.apache.spark.sql.SparkSession

/**
 * 新版本spark-hive测试
 */
object NewSparkHiveDemo {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder()
      .appName("Spark Hive Example")
      .master("local")
      .config("spark.sql.warehouse.dir", "/user/hive/warehouse/")
      .enableHiveSupport()
      .getOrCreate()

    import spark.implicits._
    import spark.sql
    sql("CREATE TABLE IF NOT EXISTS src (key INT, value STRING)")
    val data = Array((1, "val1"), (2, "val2"), (3, "val3"))
    var df = spark.createDataFrame(data).toDF("key", "value")
    df.createOrReplaceTempView("temp_src")
    sql("insert into src select key,value from temp_src")
    sql("SELECT * FROM src").show()
  }
}

hadoop fs -chmod 777 /user/hive/warehouse/