Merge branch 'master' of https://gitee.com/dingjiawen/self_example
This commit is contained in:
kevinding1125
commit
0c74438f55
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,27 @@
|
||||||
|
city_id city_name area
|
||||||
|
1 北京 华北
|
||||||
|
2 上海 华东
|
||||||
|
3 深圳 华南
|
||||||
|
4 广州 华南
|
||||||
|
5 武汉 华中
|
||||||
|
6 南京 华东
|
||||||
|
7 天津 华北
|
||||||
|
8 成都 西南
|
||||||
|
9 哈尔滨 东北
|
||||||
|
10 大连 东北
|
||||||
|
11 沈阳 东北
|
||||||
|
12 西安 西北
|
||||||
|
13 长沙 华中
|
||||||
|
14 重庆 西南
|
||||||
|
15 济南 华东
|
||||||
|
16 石家庄 华北
|
||||||
|
17 银川 西北
|
||||||
|
18 杭州 华东
|
||||||
|
19 保定 华北
|
||||||
|
20 福州 华南
|
||||||
|
21 贵阳 西南
|
||||||
|
22 青岛 华东
|
||||||
|
23 苏州 华东
|
||||||
|
24 郑州 华北
|
||||||
|
25 无锡 华东
|
||||||
|
26 厦门 华南
|
||||||
|
|
@ -0,0 +1,101 @@
|
||||||
|
product_id product_name extend_info
|
||||||
|
1 商品_1 自营
|
||||||
|
2 商品_2 自营
|
||||||
|
3 商品_3 自营
|
||||||
|
4 商品_4 自营
|
||||||
|
5 商品_5 自营
|
||||||
|
6 商品_6 自营
|
||||||
|
7 商品_7 自营
|
||||||
|
8 商品_8 自营
|
||||||
|
9 商品_9 自营
|
||||||
|
10 商品_10 自营
|
||||||
|
11 商品_11 自营
|
||||||
|
12 商品_12 第三方
|
||||||
|
13 商品_13 第三方
|
||||||
|
14 商品_14 自营
|
||||||
|
15 商品_15 自营
|
||||||
|
16 商品_16 自营
|
||||||
|
17 商品_17 第三方
|
||||||
|
18 商品_18 自营
|
||||||
|
19 商品_19 第三方
|
||||||
|
20 商品_20 自营
|
||||||
|
21 商品_21 第三方
|
||||||
|
22 商品_22 第三方
|
||||||
|
23 商品_23 自营
|
||||||
|
24 商品_24 第三方
|
||||||
|
25 商品_25 第三方
|
||||||
|
26 商品_26 自营
|
||||||
|
27 商品_27 自营
|
||||||
|
28 商品_28 自营
|
||||||
|
29 商品_29 第三方
|
||||||
|
30 商品_30 第三方
|
||||||
|
31 商品_31 自营
|
||||||
|
32 商品_32 自营
|
||||||
|
33 商品_33 自营
|
||||||
|
34 商品_34 自营
|
||||||
|
35 商品_35 第三方
|
||||||
|
36 商品_36 第三方
|
||||||
|
37 商品_37 自营
|
||||||
|
38 商品_38 自营
|
||||||
|
39 商品_39 自营
|
||||||
|
40 商品_40 第三方
|
||||||
|
41 商品_41 第三方
|
||||||
|
42 商品_42 自营
|
||||||
|
43 商品_43 自营
|
||||||
|
44 商品_44 自营
|
||||||
|
45 商品_45 自营
|
||||||
|
46 商品_46 自营
|
||||||
|
47 商品_47 自营
|
||||||
|
48 商品_48 自营
|
||||||
|
49 商品_49 自营
|
||||||
|
50 商品_50 第三方
|
||||||
|
51 商品_51 第三方
|
||||||
|
52 商品_52 自营
|
||||||
|
53 商品_53 第三方
|
||||||
|
54 商品_54 自营
|
||||||
|
55 商品_55 自营
|
||||||
|
56 商品_56 自营
|
||||||
|
57 商品_57 自营
|
||||||
|
58 商品_58 第三方
|
||||||
|
59 商品_59 自营
|
||||||
|
60 商品_60 第三方
|
||||||
|
61 商品_61 第三方
|
||||||
|
62 商品_62 自营
|
||||||
|
63 商品_63 自营
|
||||||
|
64 商品_64 自营
|
||||||
|
65 商品_65 第三方
|
||||||
|
66 商品_66 自营
|
||||||
|
67 商品_67 自营
|
||||||
|
68 商品_68 自营
|
||||||
|
69 商品_69 自营
|
||||||
|
70 商品_70 自营
|
||||||
|
71 商品_71 自营
|
||||||
|
72 商品_72 自营
|
||||||
|
73 商品_73 自营
|
||||||
|
74 商品_74 自营
|
||||||
|
75 商品_75 自营
|
||||||
|
76 商品_76 第三方
|
||||||
|
77 商品_77 自营
|
||||||
|
78 商品_78 自营
|
||||||
|
79 商品_79 第三方
|
||||||
|
80 商品_80 第三方
|
||||||
|
81 商品_81 第三方
|
||||||
|
82 商品_82 自营
|
||||||
|
83 商品_83 自营
|
||||||
|
84 商品_84 自营
|
||||||
|
85 商品_85 第三方
|
||||||
|
86 商品_86 自营
|
||||||
|
87 商品_87 自营
|
||||||
|
88 商品_88 自营
|
||||||
|
89 商品_89 自营
|
||||||
|
90 商品_90 自营
|
||||||
|
91 商品_91 自营
|
||||||
|
92 商品_92 自营
|
||||||
|
93 商品_93 第三方
|
||||||
|
94 商品_94 第三方
|
||||||
|
95 商品_95 自营
|
||||||
|
96 商品_96 自营
|
||||||
|
97 商品_97 自营
|
||||||
|
98 商品_98 自营
|
||||||
|
99 商品_99 第三方
|
||||||
|
100 商品_100 第三方
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,116 @@
|
||||||
|
package com.atguigu.spark.core.practice
|
||||||
|
|
||||||
|
import org.apache.spark.rdd.RDD
|
||||||
|
import org.apache.spark.{SparkConf, SparkContext}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 统计热门品类Top10
|
||||||
|
*/
|
||||||
|
object example1 {
|
||||||
|
|
||||||
|
|
||||||
|
def main(args: Array[String]): Unit = {
|
||||||
|
|
||||||
|
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("example1")
|
||||||
|
val sc: SparkContext = new SparkContext(sparkConf)
|
||||||
|
|
||||||
|
//2019-05-05,85,e2eef06e-beaa-4b49-acaf-38e057e1cd6e,31,2019-05-05 02:54:16,苹果,-1,-1,,,,,19
|
||||||
|
val sourceData: RDD[String] = sc.textFile("data/user_visit_action.txt")
|
||||||
|
|
||||||
|
//TODO 1.分别获取点击数，下单数，支付数
|
||||||
|
|
||||||
|
//1)获取点击数
|
||||||
|
//过滤掉不是点击事件的值
|
||||||
|
val clickFilterRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(6) != "-1"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
val clickRDD: RDD[(String, Int)] = clickFilterRDD.map {
|
||||||
|
data => {
|
||||||
|
val line: Array[String] = data.split("_")
|
||||||
|
(line(6),1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val clickNumRDD = clickRDD.reduceByKey(_ + _)
|
||||||
|
|
||||||
|
|
||||||
|
//2)获取下单数
|
||||||
|
//过滤掉不是下单事件的值
|
||||||
|
val filterOrderRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(8) != "null"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
//这里使用flatMap将合在一起的[(1,1),(2,1),(3,1)]拆开
|
||||||
|
val orderNumRDD = filterOrderRDD.flatMap(
|
||||||
|
data => {
|
||||||
|
val words: Array[String] = data.split("_")
|
||||||
|
val ids = words(8).split(",")
|
||||||
|
ids.map(id => (id, 1))
|
||||||
|
}
|
||||||
|
).reduceByKey(_+_)
|
||||||
|
|
||||||
|
|
||||||
|
//3)获取支付数
|
||||||
|
val filterPayRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(10) != "null"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
//这里使用flatMap将合在一起的[(1,1),(2,1),(3,1)]拆开
|
||||||
|
val payNumRDD = filterPayRDD.flatMap(
|
||||||
|
data => {
|
||||||
|
val words: Array[String] = data.split("_")
|
||||||
|
val ids = words(10).split(",")
|
||||||
|
ids.map(id => (id, 1))
|
||||||
|
}
|
||||||
|
).reduceByKey(_+_)
|
||||||
|
|
||||||
|
//TODO 2.将三者合并为(id,(点击数,下单数,支付数))
|
||||||
|
//采用cogroup将三者合并
|
||||||
|
val totalRDD: RDD[(String, (Iterable[Int], Iterable[Int], Iterable[Int]))] = clickNumRDD.cogroup(orderNumRDD, payNumRDD)
|
||||||
|
val combineRDD: Array[(String, (Int, Int, Int))] = totalRDD.mapValues{
|
||||||
|
case (iter1,iter2,iter3) =>{
|
||||||
|
var clickNum=0
|
||||||
|
val clickIter = iter1.iterator
|
||||||
|
if (clickIter.hasNext){
|
||||||
|
clickNum = clickIter.next()
|
||||||
|
}
|
||||||
|
|
||||||
|
var orderNum =0
|
||||||
|
val orderIter = iter2.iterator
|
||||||
|
if (orderIter.hasNext){
|
||||||
|
orderNum = orderIter.next()
|
||||||
|
}
|
||||||
|
|
||||||
|
var payNum =0
|
||||||
|
val payIter = iter3.iterator
|
||||||
|
if (payIter.hasNext){
|
||||||
|
payNum = payIter.next()
|
||||||
|
}
|
||||||
|
(clickNum,orderNum,payNum)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}.sortBy(_._2,false).take(10)
|
||||||
|
|
||||||
|
combineRDD.foreach(println(_))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,111 @@
|
||||||
|
package com.atguigu.spark.core.practice
|
||||||
|
|
||||||
|
import org.apache.spark.rdd.RDD
|
||||||
|
import org.apache.spark.{SparkConf, SparkContext}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 统计热门品类Top10
|
||||||
|
*/
|
||||||
|
object example1_other {
|
||||||
|
|
||||||
|
|
||||||
|
def main(args: Array[String]): Unit = {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 第一种方法存在的问题：
|
||||||
|
* total的复用问题
|
||||||
|
* cogroup存在的shuffle效率低问题
|
||||||
|
*/
|
||||||
|
|
||||||
|
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("example1")
|
||||||
|
val sc: SparkContext = new SparkContext(sparkConf)
|
||||||
|
|
||||||
|
//2019-05-05,85,e2eef06e-beaa-4b49-acaf-38e057e1cd6e,31,2019-05-05 02:54:16,苹果,-1,-1,,,,,19
|
||||||
|
val sourceData: RDD[String] = sc.textFile("data/user_visit_action.txt")
|
||||||
|
|
||||||
|
sourceData.cache()
|
||||||
|
|
||||||
|
//TODO 1.分别获取点击数，下单数，支付数
|
||||||
|
|
||||||
|
//1)获取点击数
|
||||||
|
//过滤掉不是点击事件的值
|
||||||
|
val clickFilterRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(6) != "-1"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
val clickRDD: RDD[(String, Int)] = clickFilterRDD.map {
|
||||||
|
data => {
|
||||||
|
val line: Array[String] = data.split("_")
|
||||||
|
(line(6),1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val clickNumRDD = clickRDD.reduceByKey(_ + _)
|
||||||
|
|
||||||
|
|
||||||
|
//2)获取下单数
|
||||||
|
//过滤掉不是下单事件的值
|
||||||
|
val filterOrderRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(8) != "null"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
//这里使用flatMap将合在一起的[(1,1),(2,1),(3,1)]拆开
|
||||||
|
val orderNumRDD = filterOrderRDD.flatMap(
|
||||||
|
data => {
|
||||||
|
val words: Array[String] = data.split("_")
|
||||||
|
val ids = words(8).split(",")
|
||||||
|
ids.map(id => (id, 1))
|
||||||
|
}
|
||||||
|
).reduceByKey(_+_)
|
||||||
|
|
||||||
|
|
||||||
|
//3)获取支付数
|
||||||
|
val filterPayRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(10) != "null"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
//这里使用flatMap将合在一起的[(1,1),(2,1),(3,1)]拆开
|
||||||
|
val payNumRDD = filterPayRDD.flatMap(
|
||||||
|
data => {
|
||||||
|
val words: Array[String] = data.split("_")
|
||||||
|
val ids = words(10).split(",")
|
||||||
|
ids.map(id => (id, 1))
|
||||||
|
}
|
||||||
|
).reduceByKey(_+_)
|
||||||
|
|
||||||
|
//TODO 2.将三者合并为(id,(点击数,下单数,支付数))
|
||||||
|
//采用cogroup将三者合并
|
||||||
|
val clickSumRDD = clickNumRDD.mapValues((_, 0, 0))
|
||||||
|
val orderSumRDD = orderNumRDD.mapValues((0, _, 0))
|
||||||
|
val paySumRDD = payNumRDD.mapValues((0, 0, _))
|
||||||
|
|
||||||
|
//此时得到的可能有重复的id,这时通过reduceByKey将其合并
|
||||||
|
val sumRDD: RDD[(String, (Int, Int, Int))] = clickSumRDD.union(orderSumRDD).union(paySumRDD)
|
||||||
|
val result = sumRDD.reduceByKey {
|
||||||
|
case (tuple1, tuple2) => {
|
||||||
|
(tuple1._1 + tuple2._1, tuple1._2 + tuple2._2, tuple1._3 + tuple2._3)
|
||||||
|
}
|
||||||
|
}.sortBy(_._2,false)
|
||||||
|
|
||||||
|
|
||||||
|
result.foreach(println(_))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,208 @@
|
||||||
|
package com.atguigu.spark.core.practice
|
||||||
|
|
||||||
|
|
||||||
|
import org.apache.spark.rdd.RDD
|
||||||
|
import org.apache.spark.util.AccumulatorV2
|
||||||
|
import org.apache.spark.{SparkConf, SparkContext}
|
||||||
|
|
||||||
|
import scala.collection.mutable
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 使用累加器实现热门品类Top10
|
||||||
|
*/
|
||||||
|
object example1_withAcc {
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 之前的使用reduceByKey虽然可以在shuffle之前进行预聚合，但是仍然不可以避免使用shuffle
|
||||||
|
* 如果能使用累加器进行计算，则可以避免效率低的shuffle操作，加快计算速度
|
||||||
|
*
|
||||||
|
* @param args
|
||||||
|
*/
|
||||||
|
def main(args: Array[String]): Unit = {
|
||||||
|
|
||||||
|
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("top10")
|
||||||
|
val sc = new SparkContext(sparkConf)
|
||||||
|
|
||||||
|
val sourceData = sc.textFile("data/user_visit_action.txt")
|
||||||
|
sourceData.cache()
|
||||||
|
|
||||||
|
//TODO 1.分别获取点击数，下单数，支付数
|
||||||
|
|
||||||
|
//1)获取点击数
|
||||||
|
//过滤掉不是点击事件的值
|
||||||
|
val clickFilterRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(6) != "-1"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
val clickRDD: RDD[(String, String)] = clickFilterRDD.map {
|
||||||
|
data => {
|
||||||
|
val line: Array[String] = data.split("_")
|
||||||
|
(line(6), "click")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
//2)获取下单数
|
||||||
|
//过滤掉不是下单事件的值
|
||||||
|
val filterOrderRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(8) != "null"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
//这里使用flatMap将合在一起的[(1,1),(2,1),(3,1)]拆开
|
||||||
|
val orderNumRDD: RDD[(String, String)] = filterOrderRDD.flatMap(
|
||||||
|
data => {
|
||||||
|
val words: Array[String] = data.split("_")
|
||||||
|
val ids = words(8).split(",")
|
||||||
|
ids.map(id => (id, "order"))
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
//3)获取支付数
|
||||||
|
val filterPayRDD = sourceData.filter(
|
||||||
|
data => {
|
||||||
|
val words = data.split("_")
|
||||||
|
words(10) != "null"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
//这里使用flatMap将合在一起的[(1,1),(2,1),(3,1)]拆开
|
||||||
|
val payNumRDD: RDD[(String, String)] = filterPayRDD.flatMap(
|
||||||
|
data => {
|
||||||
|
val words: Array[String] = data.split("_")
|
||||||
|
val ids = words(10).split(",")
|
||||||
|
ids.map(id => (id, "pay"))
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
val accumulator = new HotCategoryAccumulator
|
||||||
|
sc.register(accumulator, "hotCategoryAccumulator")
|
||||||
|
// clickRDD.union(orderNumRDD).union(payNumRDD).collect().foreach(println(_))
|
||||||
|
clickRDD.union(orderNumRDD).union(payNumRDD).foreach {
|
||||||
|
case (id, actionId) => {
|
||||||
|
accumulator.add(id, actionId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
val total: mutable.Map[String, HotCategory] = accumulator.value
|
||||||
|
total.toList.sortWith {
|
||||||
|
case (iter1, iter2) => {
|
||||||
|
val category1 = iter1._2
|
||||||
|
val category2 = iter2._2
|
||||||
|
if (category1.clickNum > category2.clickNum) {
|
||||||
|
true
|
||||||
|
} else if (category1.clickNum < category2.clickNum) {
|
||||||
|
false
|
||||||
|
} else {
|
||||||
|
if (category1.orderNum > category2.orderNum) {
|
||||||
|
true
|
||||||
|
} else if (category1.orderNum < category2.orderNum) {
|
||||||
|
false
|
||||||
|
} else {
|
||||||
|
if (category1.payNum >= category2.payNum) {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.map{
|
||||||
|
case(cid,hotCategory) =>{
|
||||||
|
|
||||||
|
(cid,(hotCategory.clickNum,hotCategory.orderNum,hotCategory.payNum))
|
||||||
|
}
|
||||||
|
|
||||||
|
}.take(10).foreach(println(_))
|
||||||
|
|
||||||
|
// println(accumulator.value)
|
||||||
|
sc.stop()
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 样例类： id,clickNum,orderNum,payNum
|
||||||
|
*
|
||||||
|
* @param cid
|
||||||
|
* @param clickNum
|
||||||
|
* @param orderNum
|
||||||
|
* @param payNum
|
||||||
|
*/
|
||||||
|
case class HotCategory(val cid: String, var clickNum: Int, var orderNum: Int, var payNum: Int)
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* /**
|
||||||
|
* 自定义累加器
|
||||||
|
* 1.继承AccumulatorV2,定义泛型
|
||||||
|
* IN:(品类ID，行为类型)
|
||||||
|
* OUT:mutable.Map[String,HotCategory]
|
||||||
|
*
|
||||||
|
* 2.重写方法(6)
|
||||||
|
* */
|
||||||
|
*/
|
||||||
|
class HotCategoryAccumulator extends AccumulatorV2[(String, String), mutable.Map[String, HotCategory]] {
|
||||||
|
|
||||||
|
private var hcMap = mutable.Map[String, HotCategory]()
|
||||||
|
|
||||||
|
override def isZero: Boolean = {
|
||||||
|
hcMap.isEmpty
|
||||||
|
}
|
||||||
|
|
||||||
|
override def copy(): AccumulatorV2[(String, String), mutable.Map[String, HotCategory]] = {
|
||||||
|
new HotCategoryAccumulator
|
||||||
|
}
|
||||||
|
|
||||||
|
override def reset(): Unit = {
|
||||||
|
hcMap.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
override def add(v: (String, String)): Unit = {
|
||||||
|
|
||||||
|
val cid = v._1
|
||||||
|
val actionId = v._2
|
||||||
|
|
||||||
|
val category = hcMap.getOrElse(cid, HotCategory(cid, 0, 0, 0))
|
||||||
|
|
||||||
|
if (actionId == "click") {
|
||||||
|
category.clickNum += 1
|
||||||
|
} else if (actionId == "order") {
|
||||||
|
category.orderNum += 1
|
||||||
|
} else if (actionId == "pay") {
|
||||||
|
category.payNum += 1
|
||||||
|
}
|
||||||
|
hcMap.update(cid, category)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
override def merge(other: AccumulatorV2[(String, String), mutable.Map[String, HotCategory]]): Unit = {
|
||||||
|
var map1 = this.hcMap
|
||||||
|
val otherMap: mutable.Map[String, HotCategory] = other.value
|
||||||
|
otherMap.foreach {
|
||||||
|
|
||||||
|
case (cid, category1) => {
|
||||||
|
val category = map1.getOrElse(cid, HotCategory(cid, 0, 0, 0))
|
||||||
|
category.clickNum += category1.clickNum
|
||||||
|
category.orderNum += category1.orderNum
|
||||||
|
category.payNum += category1.payNum
|
||||||
|
|
||||||
|
map1.update(cid, category)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override def value: mutable.Map[String, HotCategory] = hcMap
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,247 @@
|
||||||
|
package com.atguigu.spark.core.practice
|
||||||
|
|
||||||
|
import org.apache.spark.rdd.RDD
|
||||||
|
import org.apache.spark.{SparkConf, SparkContext}
|
||||||
|
import org.apache.spark.util.AccumulatorV2
|
||||||
|
|
||||||
|
import java.io
|
||||||
|
import scala.collection.mutable
|
||||||
|
import scala.util.control.Breaks.{break, breakable}
|
||||||
|
|
||||||
|
object example2 {
|
||||||
|
|
||||||
|
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("top10")
|
||||||
|
val sc = new SparkContext(sparkConf)
|
||||||
|
|
||||||
|
|
||||||
|
def main(args: Array[String]): Unit = {
|
||||||
|
|
||||||
|
|
||||||
|
val sourceData = sc.textFile("data/user_visit_action.txt")
|
||||||
|
|
||||||
|
val Top10ids = Top10Ids(sourceData = sourceData)
|
||||||
|
|
||||||
|
//TODO 1.先根据包含Top10的过滤一遍
|
||||||
|
val filterRDD = sourceData.filter {
|
||||||
|
lines => {
|
||||||
|
val words = lines.split("_")
|
||||||
|
if (words(6) != "-1") {
|
||||||
|
Top10ids.contains(words(6))
|
||||||
|
} else if (words(8) != "null") {
|
||||||
|
val ids = words(8).split(",")
|
||||||
|
ContainsAnyOne(Top10ids, ids)
|
||||||
|
} else if (words(10) != "null") {
|
||||||
|
val ids = words(10).split(",")
|
||||||
|
ContainsAnyOne(Top10ids, ids)
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// filterRDD.collect().foreach(println(_))
|
||||||
|
//将ids类型的拆开
|
||||||
|
val mapRDD: RDD[(String, String)] = filterRDD.flatMap {
|
||||||
|
line => {
|
||||||
|
val words = line.split("_")
|
||||||
|
if (words(6) != "-1") {
|
||||||
|
Array((words(6), words(2)))
|
||||||
|
} else if (words(8) != "null") {
|
||||||
|
val ids = words(8).split(",")
|
||||||
|
val tuples: Array[(String, String)] = ids.map((_, words(2)))
|
||||||
|
tuples
|
||||||
|
} else if (words(10) != "null") {
|
||||||
|
val ids = words(10).split(",")
|
||||||
|
ids.map((_, words(2)))
|
||||||
|
} else {
|
||||||
|
Array(("-1", words(2)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
//mapRDD.collect().foreach(println(_))
|
||||||
|
|
||||||
|
//TODO 3.二次过滤
|
||||||
|
val totalData = mapRDD.filter {
|
||||||
|
data => {
|
||||||
|
Top10ids.contains(data._1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//TODO 4.map成需要的形式进行计算Top10
|
||||||
|
totalData.map{
|
||||||
|
data =>{
|
||||||
|
(data._1+"/"+data._2,1)
|
||||||
|
}
|
||||||
|
}.reduceByKey(_+_)
|
||||||
|
.map{
|
||||||
|
case (data,sum) =>{
|
||||||
|
val strings = data.split("/")
|
||||||
|
(strings(0),(strings(1),sum))
|
||||||
|
}
|
||||||
|
}.groupByKey()
|
||||||
|
.mapValues {
|
||||||
|
data => {
|
||||||
|
data.toList.sortBy(_._2)(Ordering.Int.reverse).take(10)
|
||||||
|
|
||||||
|
}
|
||||||
|
}.collect().foreach(println(_))
|
||||||
|
|
||||||
|
//(20,List((22e78a14-c5eb-45fe-a67d-2ce538814d98,13), (4509c42c-3aa3-4d28-84c6-5ed27bbf2444,13), (329b966c-d61b-46ad-949a-7e37142d384a,10), (632972a4-f811-4000-b920-dc12ea803a41,10), (215bdee7-db27-458d-80f4-9088d2361a2e,10), (9bdc044f-8593-49fc-bbf0-14c28f901d42,9), (4869238b-21b0-4bf5-b455-6ac3251381ac,9), (8371a0f6-df4a-4e64-8f75-1c17db255cdf,9), (cde33446-095b-433c-927b-263ba7cd102a,9), (5e3545a0-1521-4ad6-91fe-e792c20c46da,9)))
|
||||||
|
//(19,List((fde62452-7c09-4733-9655-5bd3fb705813,14), (85157915-aa25-4a8d-8ca0-9da1ee67fa70,10), (d0398e80-ad1d-4c36-b608-6ce0de1c7c85,10), (199f8e1d-db1a-4174-b0c2-ef095aaef3ee,10), (4d93913f-a892-490d-aa58-3a74b9099e29,10), (a41bc6ea-b3e3-47ce-98af-48169da7c91b,10), (329b966c-d61b-46ad-949a-7e37142d384a,10), (d56a53ff-a23e-404b-9919-5b7ef3df2664,9), (66c96daa-0525-4e1b-ba55-d38a4b462b97,9), (5e3545a0-1521-4ad6-91fe-e792c20c46da,9)))
|
||||||
|
//(15,List((632972a4-f811-4000-b920-dc12ea803a41,13), (4509c42c-3aa3-4d28-84c6-5ed27bbf2444,11), (329b966c-d61b-46ad-949a-7e37142d384a,11),
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def ContainsAnyOne(list1:List[String],list2:Array[String]): Boolean ={
|
||||||
|
var flag :Boolean =false
|
||||||
|
breakable{
|
||||||
|
for (item <- list2){
|
||||||
|
if(list1.contains(item)){
|
||||||
|
flag=true
|
||||||
|
break()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
flag
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def Top10Ids(sourceData:RDD[String]):List[(String)] ={
|
||||||
|
val accumulator = new HotCategoryAccumulator
|
||||||
|
sc.register(accumulator, "HotCategoryAccumulator")
|
||||||
|
|
||||||
|
sourceData.foreach(
|
||||||
|
line => {
|
||||||
|
val words = line.split("_")
|
||||||
|
if (words(6) != "-1") {
|
||||||
|
accumulator.add((words(6), "click"))
|
||||||
|
}
|
||||||
|
if (words(8) != "null") {
|
||||||
|
val ids = words(8).split(",")
|
||||||
|
ids.foreach{
|
||||||
|
id =>{
|
||||||
|
accumulator.add((id, "order"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
if (words(10) != "null") {
|
||||||
|
val ids = words(10).split(",")
|
||||||
|
ids.foreach{
|
||||||
|
id =>{
|
||||||
|
accumulator.add((id, "pay"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
val value: mutable.Map[String, HotCategory] = accumulator.value
|
||||||
|
val cids: List[(String)] = value.toList.sortWith {
|
||||||
|
case (iter1, iter2) => {
|
||||||
|
val category1 = iter1._2
|
||||||
|
val category2 = iter2._2
|
||||||
|
if (category1.clickNum > category2.clickNum) {
|
||||||
|
true
|
||||||
|
} else if (category1.clickNum < category2.clickNum) {
|
||||||
|
false
|
||||||
|
} else {
|
||||||
|
if (category1.orderNum > category2.orderNum) {
|
||||||
|
true
|
||||||
|
} else if (category1.orderNum < category2.orderNum) {
|
||||||
|
false
|
||||||
|
} else {
|
||||||
|
if (category1.payNum >= category2.payNum) {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.map {
|
||||||
|
case (cid, hotCategory) => {
|
||||||
|
cid
|
||||||
|
}
|
||||||
|
}.take(10)
|
||||||
|
cids
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 样例类： id,clickNum,orderNum,payNum
|
||||||
|
*
|
||||||
|
* @param cid
|
||||||
|
* @param clickNum
|
||||||
|
* @param orderNum
|
||||||
|
* @param payNum
|
||||||
|
*/
|
||||||
|
case class HotCategory(val cid: String, var clickNum: Int, var orderNum: Int, var payNum: Int)
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* /**
|
||||||
|
* 自定义累加器
|
||||||
|
* 1.继承AccumulatorV2,定义泛型
|
||||||
|
* IN:(品类ID，行为类型)
|
||||||
|
* OUT:mutable.Map[String,HotCategory]
|
||||||
|
*
|
||||||
|
* 2.重写方法(6)
|
||||||
|
* */
|
||||||
|
*/
|
||||||
|
class HotCategoryAccumulator extends AccumulatorV2[(String, String), mutable.Map[String, HotCategory]] {
|
||||||
|
|
||||||
|
private var hcMap = mutable.Map[String, HotCategory]()
|
||||||
|
|
||||||
|
override def isZero: Boolean = {
|
||||||
|
hcMap.isEmpty
|
||||||
|
}
|
||||||
|
|
||||||
|
override def copy(): AccumulatorV2[(String, String), mutable.Map[String, HotCategory]] = {
|
||||||
|
new HotCategoryAccumulator
|
||||||
|
}
|
||||||
|
|
||||||
|
override def reset(): Unit = {
|
||||||
|
hcMap.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
override def add(v: (String, String)): Unit = {
|
||||||
|
|
||||||
|
val cid = v._1
|
||||||
|
val actionId = v._2
|
||||||
|
|
||||||
|
val category = hcMap.getOrElse(cid, HotCategory(cid, 0, 0, 0))
|
||||||
|
|
||||||
|
if (actionId == "click") {
|
||||||
|
category.clickNum += 1
|
||||||
|
} else if (actionId == "order") {
|
||||||
|
category.orderNum += 1
|
||||||
|
} else if (actionId == "pay") {
|
||||||
|
category.payNum += 1
|
||||||
|
}
|
||||||
|
hcMap.update(cid, category)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
override def merge(other: AccumulatorV2[(String, String), mutable.Map[String, HotCategory]]): Unit = {
|
||||||
|
var map1 = this.hcMap
|
||||||
|
val otherMap: mutable.Map[String, HotCategory] = other.value
|
||||||
|
otherMap.foreach {
|
||||||
|
|
||||||
|
case (cid, category1) => {
|
||||||
|
val category = map1.getOrElse(cid, HotCategory(cid, 0, 0, 0))
|
||||||
|
category.clickNum += category1.clickNum
|
||||||
|
category.orderNum += category1.orderNum
|
||||||
|
category.payNum += category1.payNum
|
||||||
|
|
||||||
|
map1.update(cid, category)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override def value: mutable.Map[String, HotCategory] = hcMap
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,158 @@
|
||||||
|
package com.atguigu.spark.core.practice
|
||||||
|
|
||||||
|
import org.apache.spark.util.AccumulatorV2
|
||||||
|
import org.apache.spark.{SparkConf, SparkContext}
|
||||||
|
|
||||||
|
import scala.collection.mutable
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 计算单挑转换率
|
||||||
|
*/
|
||||||
|
object example3 {
|
||||||
|
|
||||||
|
def main(args: Array[String]): Unit = {
|
||||||
|
|
||||||
|
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("example3")
|
||||||
|
val sc = new SparkContext(sparkConf)
|
||||||
|
|
||||||
|
val sourceData = sc.textFile("data/user_visit_action.txt")
|
||||||
|
|
||||||
|
//TODO 将Data先map成需要的形式 ((user,sessionId),(time,pageId))
|
||||||
|
//TODO 这里加入sessionId是为了避免动作不是在一个session中进行的
|
||||||
|
val mapRDD = sourceData.map {
|
||||||
|
line => {
|
||||||
|
val words = line.split("_")
|
||||||
|
((words(1),words(2)), (words(4), words(3)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mapRDD.collect().foreach(println(_))
|
||||||
|
/*
|
||||||
|
((61,de45a822-fd78-42df-a80c-ef367c66b64f),(2019-07-27 19:43:41,18))
|
||||||
|
((61,de45a822-fd78-42df-a80c-ef367c66b64f),(2019-07-27 19:43:48,9))
|
||||||
|
((61,de45a822-fd78-42df-a80c-ef367c66b64f),(2019-07-27 19:43:54,2))
|
||||||
|
*/
|
||||||
|
|
||||||
|
//TODO 先根据人和sessionId进行分组
|
||||||
|
val sortedRDD = mapRDD.groupByKey()
|
||||||
|
.mapValues {
|
||||||
|
iter => {
|
||||||
|
iter.toList.sortBy(_._1).map {
|
||||||
|
case (time, pageId) => {
|
||||||
|
pageId
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// sortedRDD.collect().foreach(println(_))
|
||||||
|
/*
|
||||||
|
((38,2c5c5546-b50f-49f7-8eb4-48a85c05e8b3),List(2, 46, 42, 16, 3, 42, 24, 27, 4, 37, 4, 34, 33, 37, 33))
|
||||||
|
((21,c2677a54-c29f-49ef-8eb3-9a3bc939c7ce),List(39, 29, 47, 21, 5))
|
||||||
|
*/
|
||||||
|
|
||||||
|
val indexRDD = sortedRDD.flatMap {
|
||||||
|
case ((userId, sessionId), list) => {
|
||||||
|
val list1 = list.takeRight(list.length - 1)
|
||||||
|
// println("list:",list)
|
||||||
|
// println("list1:",list1)
|
||||||
|
list.zip(list1)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// indexRDD.collect().foreach(println(_))
|
||||||
|
|
||||||
|
/*
|
||||||
|
map的结果：
|
||||||
|
List((2,46), (46,42), (42,16), (16,3), (3,42), (42,24), (24,27), (27,4), (4,37), (37,4), (4,34), (34,33), (33,37), (37,33))
|
||||||
|
List((39,29), (29,47), (47,21), (21,5))
|
||||||
|
flatMap的结果:
|
||||||
|
(29,12)
|
||||||
|
(12,29)
|
||||||
|
(29,27)
|
||||||
|
(27,21)
|
||||||
|
(21,30)
|
||||||
|
(30,8)
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
indexRDD.cache()
|
||||||
|
|
||||||
|
val accumulator = new MapAccumulator
|
||||||
|
sc.register(accumulator,"MapAccumulator")
|
||||||
|
//TODO 统计进入的量
|
||||||
|
indexRDD.foreach{
|
||||||
|
case tuple =>{
|
||||||
|
accumulator.add(tuple)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
val map: mutable.Map[String, Int] = accumulator.value
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
//TODO 统计出入的量
|
||||||
|
val result = indexRDD.groupBy(word => word).map {
|
||||||
|
case ((page1, page2), iter) => {
|
||||||
|
val in: Double = map.getOrElse(page1, 9999999).toDouble
|
||||||
|
val out: Double = iter.toList.length.toDouble
|
||||||
|
|
||||||
|
|
||||||
|
// println(s"in:$in out:$out")
|
||||||
|
(page1 + "-" + page2, s"${page1}-${page2}的单跳转换率为${out / in}")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result.collect().foreach(println(_))
|
||||||
|
/*
|
||||||
|
(6-18,6-18的单跳转换率为0.01723625557206538)
|
||||||
|
(45-37,45-37的单跳转换率为0.019375361480624638)
|
||||||
|
(35-49,35-49的单跳转换率为0.01752370008618213)
|
||||||
|
*/
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
sc.stop()
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
class MapAccumulator extends AccumulatorV2[(String,String),mutable.Map[String,Int]]{
|
||||||
|
|
||||||
|
private var map =mutable.Map[String,Int]()
|
||||||
|
|
||||||
|
override def isZero: Boolean = {
|
||||||
|
map.isEmpty
|
||||||
|
}
|
||||||
|
|
||||||
|
override def copy(): AccumulatorV2[(String, String), mutable.Map[String, Int]] = {
|
||||||
|
new MapAccumulator
|
||||||
|
}
|
||||||
|
|
||||||
|
override def reset(): Unit = {
|
||||||
|
map.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
override def add(v: (String, String)): Unit = {
|
||||||
|
val key = v._2
|
||||||
|
val sum = map.getOrElse(key, 0)
|
||||||
|
map.update(key,sum+1)
|
||||||
|
}
|
||||||
|
|
||||||
|
override def merge(other: AccumulatorV2[(String, String), mutable.Map[String, Int]]): Unit = {
|
||||||
|
val map1 = other.value
|
||||||
|
var maps = this.map
|
||||||
|
map1.foreach{
|
||||||
|
case (key,sum1) =>{
|
||||||
|
val sum = maps.getOrElse(key, 0)
|
||||||
|
maps.update(key,sum+sum1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
override def value: mutable.Map[String, Int] = map
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,142 @@
|
||||||
|
package com.atguigu.spark.sql.practice
|
||||||
|
|
||||||
|
import org.apache.spark.SparkConf
|
||||||
|
import org.apache.spark.sql.{Encoder, Encoders, SparkSession, functions}
|
||||||
|
import org.apache.spark.sql.expressions.Aggregator
|
||||||
|
|
||||||
|
import scala.collection.mutable
|
||||||
|
import scala.collection.mutable.ListBuffer
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 采用自定义UDAF函数,实现以下功能
|
||||||
|
* 求各个区域热门商品Top3,以及对应的各个城市的商品占比，如：
|
||||||
|
* 地区 商品名称 点击次数 城市备注
|
||||||
|
* 华北 商品A 100000 北京21.2%，天津13.2%，其他65.6%
|
||||||
|
* 华北 商品P 80200 北京63.0%，太原10%，其他27.0%
|
||||||
|
* 华北 商品M 40000 北京63.0%，太原10%，其他27.0%
|
||||||
|
* 东北 商品J 92000 大连28%，辽宁17.0%，其他 55.0%
|
||||||
|
*
|
||||||
|
*/
|
||||||
|
object example1 {
|
||||||
|
|
||||||
|
def main(args: Array[String]): Unit = {
|
||||||
|
|
||||||
|
val sparkConf = new SparkConf().setMaster("local[*]").setAppName("example1")
|
||||||
|
val spark = SparkSession.builder().enableHiveSupport().config(sparkConf).getOrCreate()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
spark.sql(
|
||||||
|
"""
|
||||||
|
|select a.*,b.product_name,c.city_name,c.area
|
||||||
|
|from user_visit_action a
|
||||||
|
|left join product_info b on a.click_product_id =b.product_id
|
||||||
|
|left join city_info c on a.city_id=c.city_id
|
||||||
|
|where click_product_id > -1
|
||||||
|
|""".stripMargin
|
||||||
|
).createOrReplaceTempView("t1")
|
||||||
|
|
||||||
|
spark.udf.register("combine", functions.udaf(new CombineAggregator()))
|
||||||
|
|
||||||
|
spark.sql(
|
||||||
|
"""
|
||||||
|
|select area,product_name,count(click_product_id) as cli_num,combine(city_name)
|
||||||
|
|from t1
|
||||||
|
|group by area,product_name
|
||||||
|
|""".stripMargin
|
||||||
|
)
|
||||||
|
|
||||||
|
//区域内对点击数量进行排行
|
||||||
|
spark.sql(
|
||||||
|
"""
|
||||||
|
|select *,rank() over(partition by area order by clickCnt desc) as rank
|
||||||
|
|from t2
|
||||||
|
|""".stripMargin).createOrReplaceTempView("t3")
|
||||||
|
|
||||||
|
|
||||||
|
//取前三名
|
||||||
|
spark.sql(
|
||||||
|
"""
|
||||||
|
|select *
|
||||||
|
|from t3 where rank <= 3
|
||||||
|
|""".stripMargin).show(false)
|
||||||
|
|
||||||
|
|
||||||
|
spark.close()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 样例类记录UDAF的中间件
|
||||||
|
*
|
||||||
|
* @param total 记录所有城市总点击量之和
|
||||||
|
* @param cityMap 记录各个城市和其对应的点击量
|
||||||
|
*/
|
||||||
|
case class Buffer(var total: Long, var cityMap: mutable.Map[String, Long])
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 自定义UDAF函数,实现聚合功能
|
||||||
|
*/
|
||||||
|
class CombineAggregator extends Aggregator[String, Buffer, String] {
|
||||||
|
override def zero: Buffer = {
|
||||||
|
Buffer(0, mutable.Map[String, Long]())
|
||||||
|
}
|
||||||
|
|
||||||
|
//分区内合并函数
|
||||||
|
override def reduce(b: Buffer, a: String): Buffer = {
|
||||||
|
val city_name = a
|
||||||
|
val total = b.total + 1
|
||||||
|
val city_sum = b.cityMap.getOrElse(city_name, 0L) + 1
|
||||||
|
b.total = total
|
||||||
|
b.cityMap.update(city_name, city_sum)
|
||||||
|
b
|
||||||
|
}
|
||||||
|
|
||||||
|
//分区间结果函数
|
||||||
|
override def merge(b1: Buffer, b2: Buffer): Buffer = {
|
||||||
|
b1.total = b1.total + b2.total
|
||||||
|
b2.cityMap.foreach {
|
||||||
|
case (city_name, sum2) => {
|
||||||
|
val sum1 = b1.cityMap.getOrElse(city_name, 0L)
|
||||||
|
b1.cityMap.update(city_name, sum1 + sum2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b2
|
||||||
|
}
|
||||||
|
|
||||||
|
//结果返回函数
|
||||||
|
override def finish(reduction: Buffer): String = {
|
||||||
|
//定义一个结果函数,用于存放变换后的结果
|
||||||
|
val resultList = new ListBuffer[String]
|
||||||
|
|
||||||
|
//首先在map中排序取前两个,如果map中数量大于2，则剩下的为其他
|
||||||
|
val needed: List[(String, Long)] = reduction.cityMap.toList.sortWith {
|
||||||
|
(left, right) => {
|
||||||
|
left._2 > right._2
|
||||||
|
}
|
||||||
|
}.take(2)
|
||||||
|
|
||||||
|
val flag = reduction.cityMap.size > 2
|
||||||
|
var rsum=0L
|
||||||
|
needed.foreach{
|
||||||
|
case (city,sum) =>{
|
||||||
|
val ratio = sum*100/reduction.total
|
||||||
|
resultList.append(s"${city}:${ratio}%")
|
||||||
|
rsum +=ratio
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (flag){
|
||||||
|
resultList.append(s"其他:${100-rsum}%")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
resultList.mkString(",")
|
||||||
|
}
|
||||||
|
|
||||||
|
override def bufferEncoder: Encoder[Buffer] = Encoders.product
|
||||||
|
|
||||||
|
override def outputEncoder: Encoder[String] = Encoders.STRING
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,127 @@
|
||||||
|
--DWT层-是DWS层在各个时间跨度上进行扩充
|
||||||
|
|
||||||
|
--开启动态分区
|
||||||
|
set hive.exec.dynamic.partition=true;
|
||||||
|
set hive.exec.dynamic.partition.mode=nonstrict;
|
||||||
|
|
||||||
|
|
||||||
|
--用户主题表
|
||||||
|
DROP TABLE IF EXISTS dwt_user_topic;
|
||||||
|
CREATE EXTERNAL TABLE dwt_user_topic
|
||||||
|
(
|
||||||
|
`user_id` STRING COMMENT '用户id',
|
||||||
|
`login_date_first` STRING COMMENT '首次活跃日期',
|
||||||
|
`login_date_last` STRING COMMENT '末次活跃日期',
|
||||||
|
`login_date_1d_count` STRING COMMENT '最近1日登录次数',
|
||||||
|
`login_last_1d_day_count` BIGINT COMMENT '最近1日登录天数',
|
||||||
|
`login_last_7d_count` BIGINT COMMENT '最近7日登录次数',
|
||||||
|
`login_last_7d_day_count` BIGINT COMMENT '最近7日登录天数',
|
||||||
|
`login_last_30d_count` BIGINT COMMENT '最近30日登录次数',
|
||||||
|
`login_last_30d_day_count` BIGINT COMMENT '最近30日登录天数',
|
||||||
|
`login_count` BIGINT COMMENT '累积登录次数',
|
||||||
|
`login_day_count` BIGINT COMMENT '累积登录天数',
|
||||||
|
`order_date_first` STRING COMMENT '首次下单时间',
|
||||||
|
`order_date_last` STRING COMMENT '末次下单时间',
|
||||||
|
`order_last_1d_count` BIGINT COMMENT '最近1日下单次数',
|
||||||
|
`order_activity_last_1d_count` BIGINT COMMENT '最近1日订单参与活动次数',
|
||||||
|
`order_activity_reduce_last_1d_amount` DECIMAL(16, 2) COMMENT '最近1日订单减免金额(活动)',
|
||||||
|
`order_coupon_last_1d_count` BIGINT COMMENT '最近1日下单用券次数',
|
||||||
|
`order_coupon_reduce_last_1d_amount` DECIMAL(16, 2) COMMENT '最近1日订单减免金额(优惠券)',
|
||||||
|
`order_last_1d_original_amount` DECIMAL(16, 2) COMMENT '最近1日原始下单金额',
|
||||||
|
`order_last_1d_final_amount` DECIMAL(16, 2) COMMENT '最近1日最终下单金额',
|
||||||
|
`order_last_7d_count` BIGINT COMMENT '最近7日下单次数',
|
||||||
|
`order_activity_last_7d_count` BIGINT COMMENT '最近7日订单参与活动次数',
|
||||||
|
`order_activity_reduce_last_7d_amount` DECIMAL(16, 2) COMMENT '最近7日订单减免金额(活动)',
|
||||||
|
`order_coupon_last_7d_count` BIGINT COMMENT '最近7日下单用券次数',
|
||||||
|
`order_coupon_reduce_last_7d_amount` DECIMAL(16, 2) COMMENT '最近7日订单减免金额(优惠券)',
|
||||||
|
`order_last_7d_original_amount` DECIMAL(16, 2) COMMENT '最近7日原始下单金额',
|
||||||
|
`order_last_7d_final_amount` DECIMAL(16, 2) COMMENT '最近7日最终下单金额',
|
||||||
|
`order_last_30d_count` BIGINT COMMENT '最近30日下单次数',
|
||||||
|
`order_activity_last_30d_count` BIGINT COMMENT '最近30日订单参与活动次数',
|
||||||
|
`order_activity_reduce_last_30d_amount` DECIMAL(16, 2) COMMENT '最近30日订单减免金额(活动)',
|
||||||
|
`order_coupon_last_30d_count` BIGINT COMMENT '最近30日下单用券次数',
|
||||||
|
`order_coupon_reduce_last_30d_amount` DECIMAL(16, 2) COMMENT '最近30日订单减免金额(优惠券)',
|
||||||
|
`order_last_30d_original_amount` DECIMAL(16, 2) COMMENT '最近30日原始下单金额',
|
||||||
|
`order_last_30d_final_amount` DECIMAL(16, 2) COMMENT '最近30日最终下单金额',
|
||||||
|
`order_count` BIGINT COMMENT '累积下单次数',
|
||||||
|
`order_activity_count` BIGINT COMMENT '累积订单参与活动次数',
|
||||||
|
`order_activity_reduce_amount` DECIMAL(16, 2) COMMENT '累积订单减免金额(活动)',
|
||||||
|
`order_coupon_count` BIGINT COMMENT '累积下单用券次数',
|
||||||
|
`order_coupon_reduce_amount` DECIMAL(16, 2) COMMENT '累积订单减免金额(优惠券)',
|
||||||
|
`order_original_amount` DECIMAL(16, 2) COMMENT '累积原始下单金额',
|
||||||
|
`order_final_amount` DECIMAL(16, 2) COMMENT '累积最终下单金额',
|
||||||
|
`payment_date_first` STRING COMMENT '首次支付时间',
|
||||||
|
`payment_date_last` STRING COMMENT '末次支付时间',
|
||||||
|
`payment_last_1d_count` BIGINT COMMENT '最近1日支付次数',
|
||||||
|
`payment_last_1d_amount` DECIMAL(16, 2) COMMENT '最近1日支付金额',
|
||||||
|
`payment_last_7d_count` BIGINT COMMENT '最近7日支付次数',
|
||||||
|
`payment_last_7d_amount` DECIMAL(16, 2) COMMENT '最近7日支付金额',
|
||||||
|
`payment_last_30d_count` BIGINT COMMENT '最近30日支付次数',
|
||||||
|
`payment_last_30d_amount` DECIMAL(16, 2) COMMENT '最近30日支付金额',
|
||||||
|
`payment_count` BIGINT COMMENT '累积支付次数',
|
||||||
|
`payment_amount` DECIMAL(16, 2) COMMENT '累积支付金额',
|
||||||
|
`refund_order_last_1d_count` BIGINT COMMENT '最近1日退单次数',
|
||||||
|
`refund_order_last_1d_num` BIGINT COMMENT '最近1日退单件数',
|
||||||
|
`refund_order_last_1d_amount` DECIMAL(16, 2) COMMENT '最近1日退单金额',
|
||||||
|
`refund_order_last_7d_count` BIGINT COMMENT '最近7日退单次数',
|
||||||
|
`refund_order_last_7d_num` BIGINT COMMENT '最近7日退单件数',
|
||||||
|
`refund_order_last_7d_amount` DECIMAL(16, 2) COMMENT '最近7日退单金额',
|
||||||
|
`refund_order_last_30d_count` BIGINT COMMENT '最近30日退单次数',
|
||||||
|
`refund_order_last_30d_num` BIGINT COMMENT '最近30日退单件数',
|
||||||
|
`refund_order_last_30d_amount` DECIMAL(16, 2) COMMENT '最近30日退单金额',
|
||||||
|
`refund_order_count` BIGINT COMMENT '累积退单次数',
|
||||||
|
`refund_order_num` BIGINT COMMENT '累积退单件数',
|
||||||
|
`refund_order_amount` DECIMAL(16, 2) COMMENT '累积退单金额',
|
||||||
|
`refund_payment_last_1d_count` BIGINT COMMENT '最近1日退款次数',
|
||||||
|
`refund_payment_last_1d_num` BIGINT COMMENT '最近1日退款件数',
|
||||||
|
`refund_payment_last_1d_amount` DECIMAL(16, 2) COMMENT '最近1日退款金额',
|
||||||
|
`refund_payment_last_7d_count` BIGINT COMMENT '最近7日退款次数',
|
||||||
|
`refund_payment_last_7d_num` BIGINT COMMENT '最近7日退款件数',
|
||||||
|
`refund_payment_last_7d_amount` DECIMAL(16, 2) COMMENT '最近7日退款金额',
|
||||||
|
`refund_payment_last_30d_count` BIGINT COMMENT '最近30日退款次数',
|
||||||
|
`refund_payment_last_30d_num` BIGINT COMMENT '最近30日退款件数',
|
||||||
|
`refund_payment_last_30d_amount` DECIMAL(16, 2) COMMENT '最近30日退款金额',
|
||||||
|
`refund_payment_count` BIGINT COMMENT '累积退款次数',
|
||||||
|
`refund_payment_num` BIGINT COMMENT '累积退款件数',
|
||||||
|
`refund_payment_amount` DECIMAL(16, 2) COMMENT '累积退款金额',
|
||||||
|
`cart_last_1d_count` BIGINT COMMENT '最近1日加入购物车次数',
|
||||||
|
`cart_last_7d_count` BIGINT COMMENT '最近7日加入购物车次数',
|
||||||
|
`cart_last_30d_count` BIGINT COMMENT '最近30日加入购物车次数',
|
||||||
|
`cart_count` BIGINT COMMENT '累积加入购物车次数',
|
||||||
|
`favor_last_1d_count` BIGINT COMMENT '最近1日收藏次数',
|
||||||
|
`favor_last_7d_count` BIGINT COMMENT '最近7日收藏次数',
|
||||||
|
`favor_last_30d_count` BIGINT COMMENT '最近30日收藏次数',
|
||||||
|
`favor_count` BIGINT COMMENT '累积收藏次数',
|
||||||
|
`coupon_last_1d_get_count` BIGINT COMMENT '最近1日领券次数',
|
||||||
|
`coupon_last_1d_using_count` BIGINT COMMENT '最近1日用券(下单)次数',
|
||||||
|
`coupon_last_1d_used_count` BIGINT COMMENT '最近1日用券(支付)次数',
|
||||||
|
`coupon_last_7d_get_count` BIGINT COMMENT '最近7日领券次数',
|
||||||
|
`coupon_last_7d_using_count` BIGINT COMMENT '最近7日用券(下单)次数',
|
||||||
|
`coupon_last_7d_used_count` BIGINT COMMENT '最近7日用券(支付)次数',
|
||||||
|
`coupon_last_30d_get_count` BIGINT COMMENT '最近30日领券次数',
|
||||||
|
`coupon_last_30d_using_count` BIGINT COMMENT '最近30日用券(下单)次数',
|
||||||
|
`coupon_last_30d_used_count` BIGINT COMMENT '最近30日用券(支付)次数',
|
||||||
|
`coupon_get_count` BIGINT COMMENT '累积领券次数',
|
||||||
|
`coupon_using_count` BIGINT COMMENT '累积用券(下单)次数',
|
||||||
|
`coupon_used_count` BIGINT COMMENT '累积用券(支付)次数',
|
||||||
|
`appraise_last_1d_good_count` BIGINT COMMENT '最近1日好评次数',
|
||||||
|
`appraise_last_1d_mid_count` BIGINT COMMENT '最近1日中评次数',
|
||||||
|
`appraise_last_1d_bad_count` BIGINT COMMENT '最近1日差评次数',
|
||||||
|
`appraise_last_1d_default_count` BIGINT COMMENT '最近1日默认评价次数',
|
||||||
|
`appraise_last_7d_good_count` BIGINT COMMENT '最近7日好评次数',
|
||||||
|
`appraise_last_7d_mid_count` BIGINT COMMENT '最近7日中评次数',
|
||||||
|
`appraise_last_7d_bad_count` BIGINT COMMENT '最近7日差评次数',
|
||||||
|
`appraise_last_7d_default_count` BIGINT COMMENT '最近7日默认评价次数',
|
||||||
|
`appraise_last_30d_good_count` BIGINT COMMENT '最近30日好评次数',
|
||||||
|
`appraise_last_30d_mid_count` BIGINT COMMENT '最近30日中评次数',
|
||||||
|
`appraise_last_30d_bad_count` BIGINT COMMENT '最近30日差评次数',
|
||||||
|
`appraise_last_30d_default_count` BIGINT COMMENT '最近30日默认评价次数',
|
||||||
|
`appraise_good_count` BIGINT COMMENT '累积好评次数',
|
||||||
|
`appraise_mid_count` BIGINT COMMENT '累积中评次数',
|
||||||
|
`appraise_bad_count` BIGINT COMMENT '累积差评次数',
|
||||||
|
`appraise_default_count` BIGINT COMMENT '累积默认评价次数'
|
||||||
|
) COMMENT '会员主题宽表'
|
||||||
|
PARTITIONED BY (`dt` STRING)
|
||||||
|
STORED AS ORC
|
||||||
|
LOCATION '/warehouse/gmall/dwt/dwt_user_topic/'
|
||||||
|
TBLPROPERTIES ("orc.compress" = "snappy");
|
||||||
|
|
@ -34,9 +34,51 @@
|
||||||
<artifactId>gson</artifactId>
|
<artifactId>gson</artifactId>
|
||||||
<version>2.8.5</version>
|
<version>2.8.5</version>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<!-- <dependency>-->
|
||||||
|
<!-- <groupId>com.cqu</groupId>-->
|
||||||
|
<!-- <artifactId>ge</artifactId>-->
|
||||||
|
<!-- <version>1.0.1</version>-->
|
||||||
|
<!-- <scope>system</scope>-->
|
||||||
|
<!-- <systemPath>${basedir}/src/main/resources/lib/HistorianServiceAPI.jar</systemPath>-->
|
||||||
|
<!-- </dependency>-->
|
||||||
|
|
||||||
|
|
||||||
|
<!-- <dependency>-->
|
||||||
|
<!-- <groupId>com.cqu</groupId>-->
|
||||||
|
<!-- <artifactId>ge</artifactId>-->
|
||||||
|
<!-- <version>1.0.0</version>-->
|
||||||
|
<!-- </dependency>-->
|
||||||
|
|
||||||
|
<!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
|
||||||
|
<!-- <dependency>-->
|
||||||
|
<!-- <groupId>com.google.code.gson</groupId>-->
|
||||||
|
<!-- <artifactId>gson</artifactId>-->
|
||||||
|
<!-- <version>2.8.5</version>-->
|
||||||
|
<!-- </dependency>-->
|
||||||
|
<!-- <dependency>-->
|
||||||
|
<!-- <groupId>junit</groupId>-->
|
||||||
|
<!-- <artifactId>junit</artifactId>-->
|
||||||
|
<!-- <version>4.12</version>-->
|
||||||
|
<!-- <scope>compile</scope>-->
|
||||||
|
<!-- </dependency>-->
|
||||||
|
|
||||||
|
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<version>3.1</version>
|
||||||
|
<configuration>
|
||||||
|
<source>1.8</source>
|
||||||
|
<target>1.8</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|
@ -42,6 +42,20 @@ import org.w3c.dom.Document;
|
||||||
import org.w3c.dom.Element;
|
import org.w3c.dom.Element;
|
||||||
import org.w3c.dom.Node;
|
import org.w3c.dom.Node;
|
||||||
import org.w3c.dom.NodeList;
|
import org.w3c.dom.NodeList;
|
||||||
|
import com.ge.ip.hds.historian.API.ArchiveService;
|
||||||
|
import com.ge.ip.hds.historian.API.ArchiveServiceImpl;
|
||||||
|
import com.ge.ip.hds.historian.API.ConfigurationManager;
|
||||||
|
import com.ge.ip.hds.historian.DataContracts.Archive;
|
||||||
|
import com.ge.ip.hds.historian.DataContracts.ArchiveStatistics;
|
||||||
|
import com.ge.ip.hds.historian.DataContracts.HistorianOperationException;
|
||||||
|
import org.junit.*;
|
||||||
|
import org.w3c.dom.Document;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.Assert.assertTrue;
|
||||||
|
|
||||||
|
//import com.ge.ip.hds.historianjavaapitest.ReadClass;
|
||||||
|
|
||||||
public class ArchiveAPITest {
|
public class ArchiveAPITest {
|
||||||
|
|
||||||
|
|
@ -149,6 +163,7 @@ public class ArchiveAPITest {
|
||||||
|
|
||||||
assertEquals(createdArchive.getName(), result.getName());
|
assertEquals(createdArchive.getName(), result.getName());
|
||||||
|
|
||||||
|
|
||||||
th.deleteArchive(createdArchive.getName());
|
th.deleteArchive(createdArchive.getName());
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
@ -210,8 +225,10 @@ public class ArchiveAPITest {
|
||||||
ArchiveServiceImpl instance = new ArchiveServiceImpl();
|
ArchiveServiceImpl instance = new ArchiveServiceImpl();
|
||||||
Archive result = instance.AddArchive(archive);
|
Archive result = instance.AddArchive(archive);
|
||||||
|
|
||||||
|
|
||||||
assertEquals(archive.getName(), result.getName());
|
assertEquals(archive.getName(), result.getName());
|
||||||
|
|
||||||
|
|
||||||
TestHelper th = new TestHelper();
|
TestHelper th = new TestHelper();
|
||||||
th.deleteArchive(archive.getName());
|
th.deleteArchive(archive.getName());
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,17 @@
|
||||||
|
package com.cqu.ge.test;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @BelongsProject: GE_Migrating_data
|
||||||
|
* @BelongsPackage: com.cqu.ge.test
|
||||||
|
* @Author: marklue
|
||||||
|
* @CreateTime: 2023/5/3 14:28
|
||||||
|
* @Description: TODO
|
||||||
|
* @Version: 1.0
|
||||||
|
*/
|
||||||
|
public class try1 {
|
||||||
|
static Try try11;
|
||||||
|
|
||||||
|
public static void main(String[] args){
|
||||||
|
try11.equals(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,264 @@
|
||||||
|
#
|
||||||
|
# A fatal error has been detected by the Java Runtime Environment:
|
||||||
|
#
|
||||||
|
# EXCEPTION_ACCESS_VIOLATION (0xc0000005) at pc=0x000000006abea148, pid=12532, tid=0x0000000000004e18
|
||||||
|
#
|
||||||
|
# JRE version: Java(TM) SE Runtime Environment (8.0_311-b11) (build 1.8.0_311-b11)
|
||||||
|
# Java VM: Java HotSpot(TM) 64-Bit Server VM (25.311-b11 mixed mode windows-amd64 compressed oops)
|
||||||
|
# Problematic frame:
|
||||||
|
# V [jvm.dll+0x19a148]
|
||||||
|
#
|
||||||
|
# Failed to write core dump. Minidumps are not enabled by default on client versions of Windows
|
||||||
|
#
|
||||||
|
# If you would like to submit a bug report, please visit:
|
||||||
|
# http://bugreport.java.com/bugreport/crash.jsp
|
||||||
|
#
|
||||||
|
|
||||||
|
--------------- T H R E A D ---------------
|
||||||
|
|
||||||
|
Current thread (0x000002d51ddc9000): JavaThread "JDWP Transport Listener: dt_socket" daemon [_thread_in_vm, id=19992, stack(0x000000ec02500000,0x000000ec02600000)]
|
||||||
|
|
||||||
|
siginfo: ExceptionCode=0xc0000005, reading address 0x000002d51d8e2018
|
||||||
|
|
||||||
|
Registers:
|
||||||
|
RAX=0x000002d51d7786d0, RBX=0x0000000000000003, RCX=0x000002d51d8e2008, RDX=0x000002d51d779200
|
||||||
|
RSP=0x000000ec025ff7b0, RBP=0x000000ec025ff829, RSI=0x00000000000000b6, RDI=0x000002d51d779268
|
||||||
|
R8 =0x000002d51d779aa8, R9 =0x00007ffedf820000, R10=0x000002d51d779211, R11=0x000002d51fea87d9
|
||||||
|
R12=0x000002d51fea87d8, R13=0x000000ec025ff8b0, R14=0x000000000000005b, R15=0x00000000000000b6
|
||||||
|
RIP=0x000000006abea148, EFLAGS=0x0000000000010202
|
||||||
|
|
||||||
|
Top of Stack: (sp=0x000000ec025ff7b0)
|
||||||
|
0x000000ec025ff7b0: 000002d51ddc9000 00000000000000b6
|
||||||
|
0x000000ec025ff7c0: 0000000000000003 000002d51ddc9000
|
||||||
|
0x000000ec025ff7d0: 000002d51d779268 000002d51d779268
|
||||||
|
0x000000ec025ff7e0: 000002d51d779268 000002d51d779268
|
||||||
|
0x000000ec025ff7f0: 000002d51ddc9000 000002d51d779268
|
||||||
|
0x000000ec025ff800: 000002d51ddc9000 000002d51d779268
|
||||||
|
0x000000ec025ff810: 000002d51ddc9000 0000005b00000058
|
||||||
|
0x000000ec025ff820: 000000b600000072 000000006ac50000
|
||||||
|
0x000000ec025ff830: 00000000000000b6 0000000000000000
|
||||||
|
0x000000ec025ff840: 0000000000000000 0000000000000072
|
||||||
|
0x000000ec025ff850: 000000ec025ff9c0 0000000000000000
|
||||||
|
0x000000ec025ff860: 0000000000000000 000000ec025ff9c8
|
||||||
|
0x000000ec025ff870: 000002d51ddc9000 000002d51d779268
|
||||||
|
0x000000ec025ff880: 0000000000000000 000000006abef64f
|
||||||
|
0x000000ec025ff890: 000000ec025ff8b0 000002d51fea87d8
|
||||||
|
0x000000ec025ff8a0: 000002d507c50a01 000002d51d779268
|
||||||
|
|
||||||
|
Instructions: (pc=0x000000006abea148)
|
||||||
|
0x000000006abea128: 10 84 d2 74 0b 41 8b 45 31 f7 d0 48 63 c8 eb 05
|
||||||
|
0x000000006abea138: 41 0f b7 4d 31 4c 8b 6d 67 48 c1 e1 05 49 03 c8
|
||||||
|
0x000000006abea148: 48 8b 49 10 44 8b 75 f3 0f b6 c1 66 c1 e0 08 66
|
||||||
|
0x000000006abea158: c1 e9 08 66 0b c1 66 41 89 44 24 01 84 d2 0f 84
|
||||||
|
|
||||||
|
|
||||||
|
Register to memory mapping:
|
||||||
|
|
||||||
|
RAX=0x000002d51d7786d0 is pointing into metadata
|
||||||
|
RBX=0x0000000000000003 is an unknown value
|
||||||
|
RCX=0x000002d51d8e2008 is an unknown value
|
||||||
|
RDX=0x000002d51d779200 is pointing into metadata
|
||||||
|
RSP=0x000000ec025ff7b0 is pointing into the stack for thread: 0x000002d51ddc9000
|
||||||
|
RBP=0x000000ec025ff829 is pointing into the stack for thread: 0x000002d51ddc9000
|
||||||
|
RSI=0x00000000000000b6 is an unknown value
|
||||||
|
RDI={method} {0x000002d51d779270} 'test' '()V' in 'com/markilue/leecode/listnode/MyLinkedList'
|
||||||
|
R8 =0x000002d51d779aa8 is pointing into metadata
|
||||||
|
R9 =0x00007ffedf820000 is an unknown value
|
||||||
|
R10=0x000002d51d779211 is pointing into metadata
|
||||||
|
R11=0x000002d51fea87d9 is an unknown value
|
||||||
|
R12=0x000002d51fea87d8 is an unknown value
|
||||||
|
R13=0x000000ec025ff8b0 is pointing into the stack for thread: 0x000002d51ddc9000
|
||||||
|
R14=0x000000000000005b is an unknown value
|
||||||
|
R15=0x00000000000000b6 is an unknown value
|
||||||
|
|
||||||
|
|
||||||
|
Stack: [0x000000ec02500000,0x000000ec02600000], sp=0x000000ec025ff7b0, free space=1021k
|
||||||
|
Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code)
|
||||||
|
V [jvm.dll+0x19a148]
|
||||||
|
V [jvm.dll+0x19f64f]
|
||||||
|
V [jvm.dll+0x3408eb]
|
||||||
|
C [jdwp.dll+0x4296]
|
||||||
|
C [jdwp.dll+0xef91]
|
||||||
|
C [jdwp.dll+0x1f4f5]
|
||||||
|
C [jdwp.dll+0x1f45e]
|
||||||
|
V [jvm.dll+0x1ba3aa]
|
||||||
|
V [jvm.dll+0x23df22]
|
||||||
|
V [jvm.dll+0x29253c]
|
||||||
|
C [ucrtbase.dll+0x21bb2]
|
||||||
|
C [KERNEL32.DLL+0x17034]
|
||||||
|
C [ntdll.dll+0x52651]
|
||||||
|
|
||||||
|
|
||||||
|
--------------- P R O C E S S ---------------
|
||||||
|
|
||||||
|
Java Threads: ( => current thread )
|
||||||
|
0x000002d51fc40800 JavaThread "Service Thread" daemon [_thread_blocked, id=21716, stack(0x000000ec02c00000,0x000000ec02d00000)]
|
||||||
|
0x000002d51fba9000 JavaThread "C1 CompilerThread3" daemon [_thread_blocked, id=14316, stack(0x000000ec02b00000,0x000000ec02c00000)]
|
||||||
|
0x000002d51fb9e800 JavaThread "C2 CompilerThread2" daemon [_thread_blocked, id=19840, stack(0x000000ec02a00000,0x000000ec02b00000)]
|
||||||
|
0x000002d51fb9d800 JavaThread "C2 CompilerThread1" daemon [_thread_blocked, id=16824, stack(0x000000ec02900000,0x000000ec02a00000)]
|
||||||
|
0x000002d51fb9b000 JavaThread "C2 CompilerThread0" daemon [_thread_blocked, id=12300, stack(0x000000ec02800000,0x000000ec02900000)]
|
||||||
|
0x000002d51faf3800 JavaThread "JDWP Command Reader" daemon [_thread_in_native, id=2456, stack(0x000000ec02700000,0x000000ec02800000)]
|
||||||
|
0x000002d51faf0800 JavaThread "JDWP Event Helper Thread" daemon [_thread_blocked, id=9096, stack(0x000000ec02600000,0x000000ec02700000)]
|
||||||
|
=>0x000002d51ddc9000 JavaThread "JDWP Transport Listener: dt_socket" daemon [_thread_in_vm, id=19992, stack(0x000000ec02500000,0x000000ec02600000)]
|
||||||
|
0x000002d51ddbc000 JavaThread "Attach Listener" daemon [_thread_blocked, id=18836, stack(0x000000ec02400000,0x000000ec02500000)]
|
||||||
|
0x000002d51dd67800 JavaThread "Signal Dispatcher" daemon [_thread_blocked, id=21684, stack(0x000000ec02300000,0x000000ec02400000)]
|
||||||
|
0x000002d51dd39000 JavaThread "Finalizer" daemon [_thread_blocked, id=10800, stack(0x000000ec02200000,0x000000ec02300000)]
|
||||||
|
0x000002d51dd30800 JavaThread "Reference Handler" daemon [_thread_blocked, id=21884, stack(0x000000ec02100000,0x000000ec02200000)]
|
||||||
|
0x000002d507b9a800 JavaThread "main" [_thread_blocked, id=16780, stack(0x000000ec01700000,0x000000ec01800000)]
|
||||||
|
|
||||||
|
Other Threads:
|
||||||
|
0x000002d51dd06800 VMThread [stack: 0x000000ec02000000,0x000000ec02100000] [id=3664]
|
||||||
|
0x000002d51fc59000 WatcherThread [stack: 0x000000ec02d00000,0x000000ec02e00000] [id=15732]
|
||||||
|
|
||||||
|
VM state:not at safepoint (normal execution)
|
||||||
|
|
||||||
|
VM Mutex/Monitor currently owned by a thread: None
|
||||||
|
|
||||||
|
heap address: 0x0000000081c00000, size: 2020 MB, Compressed Oops mode: 32-bit
|
||||||
|
Narrow klass base: 0x0000000000000000, Narrow klass shift: 3
|
||||||
|
Compressed class space size: 1073741824 Address: 0x0000000100000000
|
||||||
|
|
||||||
|
Heap:
|
||||||
|
PSYoungGen total 38400K, used 10018K [0x00000000d5f00000, 0x00000000d8980000, 0x0000000100000000)
|
||||||
|
eden space 33280K, 30% used [0x00000000d5f00000,0x00000000d68c8bd8,0x00000000d7f80000)
|
||||||
|
from space 5120K, 0% used [0x00000000d8480000,0x00000000d8480000,0x00000000d8980000)
|
||||||
|
to space 5120K, 0% used [0x00000000d7f80000,0x00000000d7f80000,0x00000000d8480000)
|
||||||
|
ParOldGen total 87552K, used 0K [0x0000000081c00000, 0x0000000087180000, 0x00000000d5f00000)
|
||||||
|
object space 87552K, 0% used [0x0000000081c00000,0x0000000081c00000,0x0000000087180000)
|
||||||
|
Metaspace used 5032K, capacity 5348K, committed 5504K, reserved 1056768K
|
||||||
|
class space used 579K, capacity 595K, committed 640K, reserved 1048576K
|
||||||
|
|
||||||
|
Card table byte_map: [0x000002d518910000,0x000002d518d10000] byte_map_base: 0x000002d518502000
|
||||||
|
|
||||||
|
Marking Bits: (ParMarkBitMap*) 0x000000006b238030
|
||||||
|
Begin Bits: [0x000002d518fc0000, 0x000002d51af50000)
|
||||||
|
End Bits: [0x000002d51af50000, 0x000002d51cee0000)
|
||||||
|
|
||||||
|
Polling page: 0x000002d507cf0000
|
||||||
|
|
||||||
|
CodeCache: size=245760Kb used=1754Kb max_used=1771Kb free=244005Kb
|
||||||
|
bounds [0x000002d509550000, 0x000002d5097c0000, 0x000002d518550000]
|
||||||
|
total_blobs=523 nmethods=259 adapters=185
|
||||||
|
compilation: enabled
|
||||||
|
|
||||||
|
Compilation events (10 events):
|
||||||
|
Event: 0.989 Thread 0x000002d51fba9000 256 3 java.io.File::isInvalid (47 bytes)
|
||||||
|
Event: 0.989 Thread 0x000002d51fba9000 nmethod 256 0x000002d5096ffdd0 code [0x000002d5096fff40, 0x000002d509700390]
|
||||||
|
Event: 0.989 Thread 0x000002d51fb9e800 257 4 sun.misc.MetaIndex::mayContain (51 bytes)
|
||||||
|
Event: 0.991 Thread 0x000002d51fb9b000 nmethod 243 0x000002d509704390 code [0x000002d5097045c0, 0x000002d509705850]
|
||||||
|
Event: 0.994 Thread 0x000002d51fba9000 258 3 java.lang.Character::charCount (12 bytes)
|
||||||
|
Event: 0.994 Thread 0x000002d51fba9000 nmethod 258 0x000002d509704010 code [0x000002d509704160, 0x000002d5097042f8]
|
||||||
|
Event: 0.997 Thread 0x000002d51fb9e800 nmethod 257 0x000002d509706e10 code [0x000002d509706f60, 0x000002d509707498]
|
||||||
|
Event: 1.000 Thread 0x000002d51fba9000 259 1 java.nio.Buffer::limit (5 bytes)
|
||||||
|
Event: 1.000 Thread 0x000002d51fba9000 nmethod 259 0x000002d509703d50 code [0x000002d509703ea0, 0x000002d509703fb8]
|
||||||
|
Event: 1.017 Thread 0x000002d51fb9d800 nmethod 253 0x000002d50970a7d0 code [0x000002d50970aa80, 0x000002d50970c1a8]
|
||||||
|
|
||||||
|
GC Heap History (0 events):
|
||||||
|
No events
|
||||||
|
|
||||||
|
Deoptimization events (0 events):
|
||||||
|
No events
|
||||||
|
|
||||||
|
Classes redefined (6 events):
|
||||||
|
Event: 120.821 Thread 0x000002d51dd06800 redefined class name=com.markilue.leecode.listnode.MyLinkedList, count=1
|
||||||
|
Event: 120.822 Thread 0x000002d51dd06800 redefined class name=com.markilue.leecode.listnode.ListNode, count=1
|
||||||
|
Event: 164.527 Thread 0x000002d51dd06800 redefined class name=com.markilue.leecode.listnode.MyLinkedList, count=2
|
||||||
|
Event: 164.528 Thread 0x000002d51dd06800 redefined class name=com.markilue.leecode.listnode.ListNode, count=2
|
||||||
|
Event: 308.152 Thread 0x000002d51dd06800 redefined class name=com.markilue.leecode.listnode.MyLinkedList, count=3
|
||||||
|
Event: 308.152 Thread 0x000002d51dd06800 redefined class name=com.markilue.leecode.listnode.ListNode, count=3
|
||||||
|
|
||||||
|
Internal exceptions (7 events):
|
||||||
|
Event: 0.110 Thread 0x000002d507b9a800 Exception <a 'java/lang/NoSuchMethodError': Method sun.misc.Unsafe.defineClass(Ljava/lang/String;[BII)Ljava/lang/Class; name or signature does not match> (0x00000000d5f07cc0) thrown at [C:\jenkins\workspace\8-2-build-windows-amd64-cygwin\jdk8u311\1894\hot
|
||||||
|
Event: 0.110 Thread 0x000002d507b9a800 Exception <a 'java/lang/NoSuchMethodError': Method sun.misc.Unsafe.prefetchRead(Ljava/lang/Object;J)V name or signature does not match> (0x00000000d5f07fa8) thrown at [C:\jenkins\workspace\8-2-build-windows-amd64-cygwin\jdk8u311\1894\hotspot\src\share\vm\
|
||||||
|
Event: 0.817 Thread 0x000002d507b9a800 Exception <a 'java/io/FileNotFoundException'> (0x00000000d62ad468) thrown at [C:\jenkins\workspace\8-2-build-windows-amd64-cygwin\jdk8u311\1894\hotspot\src\share\vm\prims\jni.cpp, line 710]
|
||||||
|
Event: 0.848 Thread 0x000002d507b9a800 Exception <a 'java/security/PrivilegedActionException'> (0x00000000d638d038) thrown at [C:\jenkins\workspace\8-2-build-windows-amd64-cygwin\jdk8u311\1894\hotspot\src\share\vm\prims\jvm.cpp, line 1523]
|
||||||
|
Event: 0.848 Thread 0x000002d507b9a800 Exception <a 'java/security/PrivilegedActionException'> (0x00000000d638d430) thrown at [C:\jenkins\workspace\8-2-build-windows-amd64-cygwin\jdk8u311\1894\hotspot\src\share\vm\prims\jvm.cpp, line 1523]
|
||||||
|
Event: 0.849 Thread 0x000002d507b9a800 Exception <a 'java/security/PrivilegedActionException'> (0x00000000d638fb28) thrown at [C:\jenkins\workspace\8-2-build-windows-amd64-cygwin\jdk8u311\1894\hotspot\src\share\vm\prims\jvm.cpp, line 1523]
|
||||||
|
Event: 0.849 Thread 0x000002d507b9a800 Exception <a 'java/security/PrivilegedActionException'> (0x00000000d638ff20) thrown at [C:\jenkins\workspace\8-2-build-windows-amd64-cygwin\jdk8u311\1894\hotspot\src\share\vm\prims\jvm.cpp, line 1523]
|
||||||
|
|
||||||
|
Events (10 events):
|
||||||
|
Event: 1527.523 Executing VM operation: GetOrSetLocal
|
||||||
|
Event: 1527.523 Executing VM operation: GetOrSetLocal done
|
||||||
|
Event: 1559.202 Executing VM operation: ChangeBreakpoints
|
||||||
|
Event: 1559.202 Executing VM operation: ChangeBreakpoints done
|
||||||
|
Event: 1559.805 Executing VM operation: ChangeBreakpoints
|
||||||
|
Event: 1559.805 Executing VM operation: ChangeBreakpoints done
|
||||||
|
Event: 1573.008 Executing VM operation: ChangeBreakpoints
|
||||||
|
Event: 1573.008 Executing VM operation: ChangeBreakpoints done
|
||||||
|
Event: 1581.175 Executing VM operation: ChangeBreakpoints
|
||||||
|
Event: 1581.176 Executing VM operation: ChangeBreakpoints done
|
||||||
|
|
||||||
|
|
||||||
|
Dynamic libraries:
|
||||||
|
0x00007ff61b6b0000 - 0x00007ff61b6f7000 E:\Java\JDK8\bin\java.exe
|
||||||
|
0x00007ffee8f90000 - 0x00007ffee9188000 C:\WINDOWS\SYSTEM32\ntdll.dll
|
||||||
|
0x00007ffee7cf0000 - 0x00007ffee7dad000 C:\WINDOWS\System32\KERNEL32.DLL
|
||||||
|
0x00007ffee6a20000 - 0x00007ffee6cee000 C:\WINDOWS\System32\KERNELBASE.dll
|
||||||
|
0x00007ffee8050000 - 0x00007ffee80fe000 C:\WINDOWS\System32\ADVAPI32.dll
|
||||||
|
0x00007ffee7560000 - 0x00007ffee75fe000 C:\WINDOWS\System32\msvcrt.dll
|
||||||
|
0x00007ffee8dd0000 - 0x00007ffee8e6c000 C:\WINDOWS\System32\sechost.dll
|
||||||
|
0x00007ffee8990000 - 0x00007ffee8ab5000 C:\WINDOWS\System32\RPCRT4.dll
|
||||||
|
0x00007ffee7eb0000 - 0x00007ffee8050000 C:\WINDOWS\System32\USER32.dll
|
||||||
|
0x00007ffee6880000 - 0x00007ffee68a2000 C:\WINDOWS\System32\win32u.dll
|
||||||
|
0x00007ffee8170000 - 0x00007ffee819a000 C:\WINDOWS\System32\GDI32.dll
|
||||||
|
0x00007ffee6e40000 - 0x00007ffee6f4b000 C:\WINDOWS\System32\gdi32full.dll
|
||||||
|
0x00007ffee6f50000 - 0x00007ffee6fed000 C:\WINDOWS\System32\msvcp_win.dll
|
||||||
|
0x00007ffee6cf0000 - 0x00007ffee6df0000 C:\WINDOWS\System32\ucrtbase.dll
|
||||||
|
0x00007ffed59c0000 - 0x00007ffed5c5a000 C:\WINDOWS\WinSxS\amd64_microsoft.windows.common-controls_6595b64144ccf1df_6.0.19041.1110_none_60b5254171f9507e\COMCTL32.dll
|
||||||
|
0x00007ffee88f0000 - 0x00007ffee8920000 C:\WINDOWS\System32\IMM32.DLL
|
||||||
|
0x00007ffedf820000 - 0x00007ffedf835000 E:\Java\JDK8\jre\bin\vcruntime140.dll
|
||||||
|
0x00007ffebd920000 - 0x00007ffebd9bb000 E:\Java\JDK8\jre\bin\msvcp140.dll
|
||||||
|
0x000000006aa50000 - 0x000000006b2b0000 E:\Java\JDK8\jre\bin\server\jvm.dll
|
||||||
|
0x00007ffee7ce0000 - 0x00007ffee7ce8000 C:\WINDOWS\System32\PSAPI.DLL
|
||||||
|
0x00007ffed2a20000 - 0x00007ffed2a29000 C:\WINDOWS\SYSTEM32\WSOCK32.dll
|
||||||
|
0x00007ffed5e10000 - 0x00007ffed5e37000 C:\WINDOWS\SYSTEM32\WINMM.dll
|
||||||
|
0x00007ffedfb10000 - 0x00007ffedfb1a000 C:\WINDOWS\SYSTEM32\VERSION.dll
|
||||||
|
0x00007ffee8100000 - 0x00007ffee816b000 C:\WINDOWS\System32\WS2_32.dll
|
||||||
|
0x00007ffee4f90000 - 0x00007ffee4fa2000 C:\WINDOWS\SYSTEM32\kernel.appcore.dll
|
||||||
|
0x00007ffee0460000 - 0x00007ffee0470000 E:\Java\JDK8\jre\bin\verify.dll
|
||||||
|
0x00007ffedf7f0000 - 0x00007ffedf81b000 E:\Java\JDK8\jre\bin\java.dll
|
||||||
|
0x00007ffedb7d0000 - 0x00007ffedb806000 E:\Java\JDK8\jre\bin\jdwp.dll
|
||||||
|
0x00007ffee1c90000 - 0x00007ffee1c99000 E:\Java\JDK8\jre\bin\npt.dll
|
||||||
|
0x00007ffedf8f0000 - 0x00007ffedf920000 E:\Java\JDK8\jre\bin\instrument.dll
|
||||||
|
0x00007ffedef10000 - 0x00007ffedef28000 E:\Java\JDK8\jre\bin\zip.dll
|
||||||
|
0x00007ffee81a0000 - 0x00007ffee88e4000 C:\WINDOWS\System32\SHELL32.dll
|
||||||
|
0x00007ffee4790000 - 0x00007ffee4f24000 C:\WINDOWS\SYSTEM32\windows.storage.dll
|
||||||
|
0x00007ffee6ff0000 - 0x00007ffee7344000 C:\WINDOWS\System32\combase.dll
|
||||||
|
0x00007ffee6110000 - 0x00007ffee6140000 C:\WINDOWS\SYSTEM32\Wldp.dll
|
||||||
|
0x00007ffee7a70000 - 0x00007ffee7b1d000 C:\WINDOWS\System32\SHCORE.dll
|
||||||
|
0x00007ffee7e30000 - 0x00007ffee7e85000 C:\WINDOWS\System32\shlwapi.dll
|
||||||
|
0x00007ffee65f0000 - 0x00007ffee660f000 C:\WINDOWS\SYSTEM32\profapi.dll
|
||||||
|
0x00007ffedf850000 - 0x00007ffedf85a000 E:\Java\JDK8\jre\bin\dt_socket.dll
|
||||||
|
0x00007ffee5e70000 - 0x00007ffee5eda000 C:\WINDOWS\system32\mswsock.dll
|
||||||
|
0x00007ffee44a0000 - 0x00007ffee4684000 C:\WINDOWS\SYSTEM32\dbghelp.dll
|
||||||
|
0x00007ffee68e0000 - 0x00007ffee6962000 C:\WINDOWS\System32\bcryptPrimitives.dll
|
||||||
|
|
||||||
|
VM Arguments:
|
||||||
|
jvm_args: -agentlib:jdwp=transport=dt_socket,address=127.0.0.1:5541,suspend=y,server=n -ea -Didea.test.cyclic.buffer.size=1048576 -javaagent:C:\Users\marklue\AppData\Local\JetBrains\IntelliJIdea2021.1\captureAgent\debugger-agent.jar -Dfile.encoding=UTF-8
|
||||||
|
java_command: com.intellij.rt.junit.JUnitStarter -ideVersion5 -junit4 com.markilue.leecode.listnode.MyLinkedList,test
|
||||||
|
java_class_path (initial): D:\software\JetBrains\IntelliJ IDEA 2021.1\lib\idea_rt.jar;D:\software\JetBrains\IntelliJ IDEA 2021.1\plugins\junit\lib\junit5-rt.jar;D:\software\JetBrains\IntelliJ IDEA 2021.1\plugins\junit\lib\junit-rt.jar;E:\Java\JDK8\jre\lib\charsets.jar;E:\Java\JDK8\jre\lib\deploy.jar;E:\Java\JDK8\jre\lib\ext\access-bridge-64.jar;E:\Java\JDK8\jre\lib\ext\cldrdata.jar;E:\Java\JDK8\jre\lib\ext\dnsns.jar;E:\Java\JDK8\jre\lib\ext\jaccess.jar;E:\Java\JDK8\jre\lib\ext\jfxrt.jar;E:\Java\JDK8\jre\lib\ext\localedata.jar;E:\Java\JDK8\jre\lib\ext\nashorn.jar;E:\Java\JDK8\jre\lib\ext\sunec.jar;E:\Java\JDK8\jre\lib\ext\sunjce_provider.jar;E:\Java\JDK8\jre\lib\ext\sunmscapi.jar;E:\Java\JDK8\jre\lib\ext\sunpkcs11.jar;E:\Java\JDK8\jre\lib\ext\zipfs.jar;E:\Java\JDK8\jre\lib\javaws.jar;E:\Java\JDK8\jre\lib\jce.jar;E:\Java\JDK8\jre\lib\jfr.jar;E:\Java\JDK8\jre\lib\jfxswt.jar;E:\Java\JDK8\jre\lib\jsse.jar;E:\Java\JDK8\jre\lib\management-agent.jar;E:\Java\JDK8\jre\lib\plugin.jar;E:\Java\JDK8\jre\lib\resources.jar;E:\Java\JDK8\jre\lib\rt.jar;D:\example\self_example\Leecode\target\classes;E:\maven\apache-maven-3.5.4-bin\RepMaven\org\projectlombok\lombok\1.18.24\lombok-1.18.24.jar;E:\maven\apache-maven-3.5.4-bin\RepMaven\junit\junit\4.13.2\junit-4.13.2.jar;E:\maven\apache-maven-3.5.4-bin\RepMaven\org\hamcrest\hamcrest-core\1.3\hamcrest-core-1.3.jar;C:\Users\marklue\AppData\Local\JetBrains\IntelliJIdea2021.1\captureAgent\debugger-agent.jar
|
||||||
|
Launcher Type: SUN_STANDARD
|
||||||
|
|
||||||
|
Environment Variables:
|
||||||
|
JAVA_HOME=E:\Java\JDK8
|
||||||
|
PATH=C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\WINDOWS\System32\WindowsPowerShell\v1.0\;C:\WINDOWS\System32\OpenSSH\;D:\software\RAR½âѹ¹¤¾ß\Bandizip\;D:\;oftware\nodejs\;E:\Java\JDK8\bin;E:\maven\apache-maven-3.5.4-bin\apache-maven-3.5.4\bin;E:\scala\scala-2.12.11\bin;D:\software\anaconda\pkgs\python-3.7.11-h6244533_0;D:\software\anaconda\Scripts;D:\software\Git\Git\cmd;D:\software\nodejs;C:\Users\marklue\AppData\Local\Microsoft\WindowsApps;C:\Users\marklue\AppData\Roaming\npm;D:\software\JetBrains\PyCharm 2020.1\bin;
|
||||||
|
USERNAME=marklue
|
||||||
|
OS=Windows_NT
|
||||||
|
PROCESSOR_IDENTIFIER=Intel64 Family 6 Model 142 Stepping 10, GenuineIntel
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
--------------- S Y S T E M ---------------
|
||||||
|
|
||||||
|
OS: Windows 10.0 , 64 bit Build 19041 (10.0.19041.1806)
|
||||||
|
|
||||||
|
CPU:total 8 (initial active 8) (4 cores per cpu, 2 threads per core) family 6 model 142 stepping 10, cmov, cx8, fxsr, mmx, sse, sse2, sse3, ssse3, sse4.1, sse4.2, popcnt, avx, avx2, aes, clmul, erms, 3dnowpref, lzcnt, ht, tsc, tscinvbit, bmi1, bmi2, adx
|
||||||
|
|
||||||
|
Memory: 4k page, physical 8272104k(2053676k free), swap 11902816k(1707060k free)
|
||||||
|
|
||||||
|
vm_info: Java HotSpot(TM) 64-Bit Server VM (25.311-b11) for windows-amd64 JRE (1.8.0_311-b11), built on Sep 27 2021 05:15:14 by "java_re" with MS VC++ 15.9 (VS2017)
|
||||||
|
|
||||||
|
time: Mon Sep 5 12:31:48 2022
|
||||||
|
timezone: Öйú±ê׼ʱ¼ä
|
||||||
|
elapsed time: 1581.277238 seconds (0d 0h 26m 21s)
|
||||||
|
|
||||||
|
|
@ -0,0 +1,67 @@
|
||||||
|
package com.markilue.leecode.sort;
|
||||||
|
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @BelongsProject: Leecode
|
||||||
|
* @BelongsPackage: com.markilue.leecode.sort
|
||||||
|
* @Author: marklue
|
||||||
|
* @CreateTime: 2023/4/23 20:27
|
||||||
|
* @Description: TODO
|
||||||
|
* @Version: 1.0
|
||||||
|
*/
|
||||||
|
public class QuickSort1 {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void test(){
|
||||||
|
int[] nums={8,5,7,6,9,2,4,3,1};
|
||||||
|
quickSort(nums);
|
||||||
|
System.out.println(Arrays.toString(nums));
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
public void quickSort(int[] nums) {
|
||||||
|
quickSort(nums,0,nums.length-1);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void quickSort(int[] nums, int left, int right) {
|
||||||
|
if (left > right) return;
|
||||||
|
int partition = partition(nums, left, right);
|
||||||
|
quickSort(nums, left, partition-1);
|
||||||
|
quickSort(nums,partition+1,right);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public int partition(int[] nums, int left, int right) {
|
||||||
|
|
||||||
|
if (left > right) return left;
|
||||||
|
|
||||||
|
int start = left;
|
||||||
|
int end = right + 1;
|
||||||
|
|
||||||
|
while (start < end) {
|
||||||
|
while (start < right && nums[++start] < nums[left]) {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
while (end > left && nums[--end] > nums[left]) {
|
||||||
|
|
||||||
|
}
|
||||||
|
if (start >= end) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
swap(nums, start, end);
|
||||||
|
}
|
||||||
|
swap(nums,left,end);
|
||||||
|
|
||||||
|
return end;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void swap(int[] nums, int start, int end) {
|
||||||
|
int temp = nums[start];
|
||||||
|
nums[start] = nums[end];
|
||||||
|
nums[end] = temp;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>com.makrilue.interview</groupId>
|
||||||
|
<artifactId>RedCampus</artifactId>
|
||||||
|
<version>1.0-SNAPSHOT</version>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.13.2</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.13.2</version>
|
||||||
|
<scope>compile</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<maven.compiler.source>8</maven.compiler.source>
|
||||||
|
<maven.compiler.target>8</maven.compiler.target>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
</project>
|
||||||
|
|
@ -0,0 +1,64 @@
|
||||||
|
import java.util.Scanner;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @BelongsProject: RedCampus
|
||||||
|
* @BelongsPackage: com.markilue.interview
|
||||||
|
* @Author: marklue
|
||||||
|
* @CreateTime: 2023/3/26 15:12
|
||||||
|
* @Description: TODO
|
||||||
|
* @Version: 1.0
|
||||||
|
*/
|
||||||
|
public class Test1 {
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
Scanner sc = new Scanner(System.in);
|
||||||
|
int n = sc.nextInt();
|
||||||
|
int k = sc.nextInt();
|
||||||
|
int[] nums = new int[n];
|
||||||
|
for (int i = 0; i < n; i++) {
|
||||||
|
nums[i] = sc.nextInt();
|
||||||
|
}
|
||||||
|
|
||||||
|
int[][][] dp = new int[n + 1][k+1][2];
|
||||||
|
for (int j = 0; j < k; j++) {
|
||||||
|
dp[1][j][0] = nums[0];
|
||||||
|
dp[1][j][1] = last(nums[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
for (int i = 2; i < dp.length; i++) {
|
||||||
|
for (int j = 0; j < k; j++) {
|
||||||
|
dp[i][j][0] = Math.min(dp[i - 1][j][0], dp[i - 1][j][1])+nums[i-1];
|
||||||
|
if (j > 0) {
|
||||||
|
dp[i][j][1] = Math.min(dp[i - 1][j][0] + nums[i - 1], dp[i - 1][j - 1][1] + last(nums[i - 1]));
|
||||||
|
} else {
|
||||||
|
dp[i][j][1] = dp[i - 1][j][0] + last(nums[i - 1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.println(dp);
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int last(int num) {
|
||||||
|
|
||||||
|
for (int i = num; i > 1; i--) {
|
||||||
|
if (num % i == 0 && isProdiom(i)) {
|
||||||
|
return num / i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return num;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean isProdiom(int num) {
|
||||||
|
if (num == 1) return false;
|
||||||
|
|
||||||
|
for (int i = 2; i < Math.sqrt(num); i++) {
|
||||||
|
if (num % i == 0) return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue