admin管理员组

文章数量:1026989

env: Windows + idea + maven + scala frame

i am studying how to use spark and just trying to run code as follow...

import .apache.spark.{SparkConf, SparkContext}

object wordCount {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[*]").setAppName("WordCount")

    val sc = new SparkContext(conf)

    sc.setLogLevel("WARN")

    val fileRDD = sc.textFile("datas/words")

    val wordRDD = fileRDD.flatMap(_.split(" "))

    val word2OneRDD = wordRDD.map((_,1))

    val word2CountRDD = word2OneRDD.reduceByKey(_ + _)

    word2CountRDD.foreach(println)

    val collect = word2CountRDD.collect().toBuffer
    collect.foreach(println)

    word2CountRDD.repartition(1).saveAsTextFile("datas/words/result")

    println("save to datas/words/result")

    sc.stop()

    println("done")
  }
}

and here is the pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns=".0.0"
         xmlns:xsi=";
         xsi:schemaLocation=".0.0 .0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>.example</groupId>
    <artifactId>untitled1</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>.apache.spark</groupId>
            <artifactId>spark-2.12</artifactId>
            <version>3.3.0</version>
        </dependency>
    </dependencies>
</project>

i guess the problem cause by hadoop configured by maven on windows but can't deal with it

env: Windows + idea + maven + scala frame

i am studying how to use spark and just trying to run code as follow...

import .apache.spark.{SparkConf, SparkContext}

object wordCount {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setMaster("local[*]").setAppName("WordCount")

    val sc = new SparkContext(conf)

    sc.setLogLevel("WARN")

    val fileRDD = sc.textFile("datas/words")

    val wordRDD = fileRDD.flatMap(_.split(" "))

    val word2OneRDD = wordRDD.map((_,1))

    val word2CountRDD = word2OneRDD.reduceByKey(_ + _)

    word2CountRDD.foreach(println)

    val collect = word2CountRDD.collect().toBuffer
    collect.foreach(println)

    word2CountRDD.repartition(1).saveAsTextFile("datas/words/result")

    println("save to datas/words/result")

    sc.stop()

    println("done")
  }
}

and here is the pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns=".0.0"
         xmlns:xsi=";
         xsi:schemaLocation=".0.0 .0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>.example</groupId>
    <artifactId>untitled1</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <dependency>
            <groupId>.apache.spark</groupId>
            <artifactId>spark-2.12</artifactId>
            <version>3.3.0</version>
        </dependency>
    </dependencies>
</project>

i guess the problem cause by hadoop configured by maven on windows but can't deal with it

本文标签：

版权声明：本文标题：scala - Exception in thread &quot;main&quot; java.lang.UnsupportedOperationException: getSubject is supported only if a 内容由热心网友自发贡献，该文观点仅代表作者本人， 转载请联系作者并注明出处：http://it.en369.cn/questions/1745600622a2158435.html， 本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌抄袭侵权/违法违规的内容，一经查实，本站将立刻删除。