diff --git a/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/AAA.scala b/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/AAA.scala new file mode 100644 index 0000000..89f2e4f --- /dev/null +++ b/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/AAA.scala @@ -0,0 +1,20 @@ +package com.atguigu.spark.core.rdd.operator.transform + +import org.apache.spark.api.java.function.FilterFunction + +/** + * @ClassName aaa.java + * @author dingjiawen@xiaomi.com + * @version 1.0.0 + * @Description TODO + * @createTime 2023-07-19 18:44:00 + */ +class AAA(bbb: Int) extends FilterFunction[Int] { + + val cc = bbb + + override def call(t: Int): Boolean = { + println(cc) + return true + } +} diff --git a/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/CCC.scala b/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/CCC.scala new file mode 100644 index 0000000..0073af1 --- /dev/null +++ b/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/CCC.scala @@ -0,0 +1,18 @@ +package com.atguigu.spark.core.rdd.operator.transform + +/** + * @ClassName aaa.java + * @author dingjiawen@xiaomi.com + * @version 1.0.0 + * @Description TODO + * @createTime 2023-07-19 18:44:00 + */ +class CCC(bbb: Int) extends (Int => Boolean) with Serializable { + + val cc = bbb + + override def apply(t: Int): Boolean = { + println(cc) + return true + } +} diff --git a/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/HdfsToHolo_MessageSampler1.scala b/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/HdfsToHolo_MessageSampler1.scala new file mode 100644 index 0000000..d080771 --- /dev/null +++ b/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/HdfsToHolo_MessageSampler1.scala @@ -0,0 +1,69 @@ +package com.atguigu.spark.core.rdd.operator.transform + +import com.fasterxml.jackson.databind.ObjectMapper +import net.jpountz.xxhash.{XXHash64, XXHashFactory} +import org.apache.log4j.Logger +import org.apache.spark.api.java.function.FilterFunction + +import java.nio.ByteBuffer + +/** + * created by likunyi@xiaomi.com + * at 2022-04-29 15:32:00 + * 该算子会对收到的数据进行抽样, 留下一部分数据, 丢掉另一部分数据 + * + * samplingRate是采样率, 只能是整数 + * 当samplingRate = 10时, 代表采样率是10% + * 当samplingRate = 64时, 代表采样率是64% + */ +class HdfsToHolo_MessageSampler1(samplingRate: Int) extends FilterFunction[String] { + + @transient private var objectMapper: ObjectMapper = new ObjectMapper() + @transient private val greatestCommonDivisor = 100 + @transient private val NUMERATOR: Int = samplingRate / greatestCommonDivisor + @transient private val DENOMINATOR: Int = 100 / greatestCommonDivisor + @transient private val XXHASH_SEED: Long = 0x9747b28c + @transient private val hasher: XXHash64 = XXHashFactory.fastestInstance().hash64() + @transient private val logger: Logger = Logger.getLogger(this.getClass.getName) + + //初始化各项参数 + // def open(): Tuple5[ObjectMapper, Int, Int, Long, XXHash64] = { + // + // objectMapper = new ObjectMapper() + // + // val numerator = samplingRate // 分子就是采样率 + // val denominator = 100 // 分母永远是100 + // val greatestCommonDivisor = MathUtils.getGCD(numerator, denominator) + // + // NUMERATOR = numerator / greatestCommonDivisor + // DENOMINATOR = denominator / greatestCommonDivisor + // XXHASH_SEED = 0x9747b28c + // hasher = XXHashFactory.fastestInstance().hash64() + // (objectMapper,NUMERATOR,DENOMINATOR,XXHASH_SEED,hasher) + // } // open + + override def call(input: String): Boolean = { + logger.info(s"所有参数：【objectMapper：${objectMapper}】，【NUMERATOR：${NUMERATOR}】，【DENOMINATOR：${DENOMINATOR}】，【hasher：${hasher}】") + logger.info(s"${input}") + + + // 如果不抽样(即抽样率是100%), 则直接将数据传递给下游, 不做任何处理 + if (samplingRate == 100) { + return true + } + + val currentMessage = objectMapper.readTree(input) + + // 如需抽样(即抽样率不是100%), 则使用distinct_id的哈希值去做抽样 + if (hash(currentMessage.get("distinct_id").asText()) % DENOMINATOR < NUMERATOR) { // 粗略地讲: 若采样率是64%, 则就将distinct_id的哈希值对100取余, 余数为0到63的都留下, 余数为64到99的都扔掉 + return true + } else { + return false + } + } + + private def hash(distinct_id: String): Long = { + Math.abs(hasher.hash(ByteBuffer.wrap(distinct_id.getBytes("UTF-8")), XXHASH_SEED)) + } // hash + +} diff --git a/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/Spark08_RDD_Operator_Transform.scala b/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/Spark08_RDD_Operator_Transform.scala index 82c0f10..14a7204 100644 --- a/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/Spark08_RDD_Operator_Transform.scala +++ b/Big_data_example/Spark/src/main/java/com/atguigu/spark/core/rdd/operator/transform/Spark08_RDD_Operator_Transform.scala @@ -1,9 +1,5 @@ package com.atguigu.spark.core.rdd.operator.transform -import java.text.SimpleDateFormat -import java.util.Date - -import org.apache.spark.rdd.RDD import org.apache.spark.{SparkConf, SparkContext} /** @@ -11,21 +7,25 @@ import org.apache.spark.{SparkConf, SparkContext} */ object Spark08_RDD_Operator_Transform { - def main(args: Array[String]): Unit = { + def main(args: Array[String]): Unit = { - val sparkConf = new SparkConf().setMaster("local[*]").setAppName("Operator") - //创建上下文环境对象 - val sc =new SparkContext(sparkConf) + val sparkConf = new SparkConf().setMaster("local[*]").setAppName("Operator") + //创建上下文环境对象 + val sc = new SparkContext(sparkConf) - //TODO 算子 - filter - val rdd = sc.makeRDD( - List(1,2,3,4) - ) + //TODO 算子 - filter + val rdd = sc.makeRDD( + List(1, 2, 3, 4) + ) - val filterRDD = rdd.filter(_ % 2 == 0) + val filterRDD = rdd.filter( + new CCC(10) + ) - filterRDD.collect().foreach(println(_)) + // val filterRDD = rdd.filter(_ % 2 == 0) - sc.stop() - } + filterRDD.collect().foreach(println(_)) + + sc.stop() + } } diff --git a/Big_data_example/rt-gmall-parent/gmall-realtime/src/main/java/com/atguigu/gmall/realtime/app/dwm/GenerateRPN.scala b/Big_data_example/rt-gmall-parent/gmall-realtime/src/main/java/com/atguigu/gmall/realtime/app/dwm/GenerateRPN.scala new file mode 100644 index 0000000..b622dc9 --- /dev/null +++ b/Big_data_example/rt-gmall-parent/gmall-realtime/src/main/java/com/atguigu/gmall/realtime/app/dwm/GenerateRPN.scala @@ -0,0 +1,102 @@ +package com.atguigu.gmall.realtime.app.dwm + +import scala.collection.mutable + +/** + * @ClassName GenerateRPN.java + * @author dingjiawen@xiaomi.com + * @version 1.0.0 + * @Description TODO + * @createTime 2023-06-30 17:27:00 + */ +object GenerateRPN { + + def main(args: Array[String]): Unit = { + //测试用例 + //String str = "1+2*3-4*5-6+7*8-9"; //123*+45*-6-78*+9- +// var str = "1 + 2 * 3 - 4 * 5 - (6 + 7 * 8 - 9)"; //123*+45*-6-78*+9- +// var str = "5 + 2 * 3"; //123*+45*-6-78*+9- + var str = "6 * ( 5 + ( 2 + 3 ) * 8 + 3 )"; //6523+8*+3+* + var RPNStack: Array[String] = generateRPN(str) + println(RPNStack.mkString(",")) + println(evalRPN(RPNStack)) + } + + def generateRPN(expression: String): Array[String] = { + val precedence = Map("+" -> 1, "-" -> 1, "*" -> 2, "/" -> 2) + val output = mutable.Stack[String]() + val stack = mutable.Stack[String]() + + def isOperator(token: String): Boolean = { + precedence.contains(token) } + + def hasHigherPrecedence(op1: String, op2: String): Boolean = { + precedence(op1) >= precedence(op2) + } + + def processOperator(op: String): Unit = { + while (stack.nonEmpty && isOperator(stack.top) && hasHigherPrecedence(stack.top, op)) { + output.push(stack.pop()) + } + stack.push(op) + } + + def processOperand(operand: String): Unit = { + output.push(operand) + } + + def processParenthesis(): Unit = { + while (stack.nonEmpty && stack.top != "(") { + output.push(stack.pop()) + } + stack.pop() // 弹出左括号 + } + + for (token <- expression.split("\\s+")) { + token match { + case "(" => stack.push(token) + case ")" => processParenthesis() + case t if isOperator(t) => processOperator(t) + case _ => processOperand(token) + } + } + + while (stack.nonEmpty) { + output.push(stack.pop()) + } + + output.toArray.reverse + } + + def evalRPN(tokens: Array[String]): Int = { + val stack = mutable.Stack[Int]() + + for (token <- tokens) { + if (isOperator(token)) { + val operand2 = stack.pop() + val operand1 = stack.pop() + val result = performOperation(token, operand1, operand2) + stack.push(result) + } else { + stack.push(token.toInt) + } + } + + stack.pop() + } + + def isOperator(token: String): Boolean = { + token == "+" || token == "-" || token == "*" || token == "/" + } + + def performOperation(operator: String, operand1: Int, operand2: Int): Int = { + operator match { + case "+" => operand1 + operand2 + case "-" => operand1 - operand2 + case "*" => operand1 * operand2 + case "/" => operand1 / operand2 + } + } + + +} diff --git a/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter03/Scala02_Oper.scala b/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter03/Scala02_Oper.scala index ab3f12f..7b028e2 100644 --- a/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter03/Scala02_Oper.scala +++ b/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter03/Scala02_Oper.scala @@ -1,6 +1,6 @@ package com.atguigu.scala.chapter03 -import com.atguigu.scala.test.User +import com.atguigu.scala.test1.User object Scala02_Oper { diff --git a/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter06/Scala09_Object_Instance_4.scala b/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter06/Scala09_Object_Instance_4.scala index 9e8a9f5..a9dc5bd 100644 --- a/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter06/Scala09_Object_Instance_4.scala +++ b/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter06/Scala09_Object_Instance_4.scala @@ -1,6 +1,6 @@ package com.atguigu.scala.chapter06 -import com.atguigu.scala.test.ScalaUser +import com.atguigu.scala.test1.ScalaUser object Scala09_Object_Instance_4 { diff --git a/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter06/Scala09_Object_Instance_5.scala b/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter06/Scala09_Object_Instance_5.scala index 55f6f31..098ec2e 100644 --- a/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter06/Scala09_Object_Instance_5.scala +++ b/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter06/Scala09_Object_Instance_5.scala @@ -1,6 +1,6 @@ package com.atguigu.scala.chapter06 -import com.atguigu.scala.test.ScalaUser +import com.atguigu.scala.test1.ScalaUser object Scala09_Object_Instance_5 { diff --git a/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter10/Scala05_Transform.scala b/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter10/Scala05_Transform.scala index 829b6e6..cebf9c8 100644 --- a/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter10/Scala05_Transform.scala +++ b/Big_data_example/scala/scala0224/src/main/java/com/atguigu/scala/chapter10/Scala05_Transform.scala @@ -14,7 +14,7 @@ object Scala05_Transform extends Parent with MyTrait { //TODO 3.特征或伴生对象 //TODO 4.其他地方声明(包对象) //TODO 5.直接导入 - import com.atguigu.scala.test.TestTransform._ + import com.atguigu.scala.test1.TestTransform._ val user=new User() user.insertUser() user.updateUser() diff --git a/Leecode/src/main/java/com/markilue/leecode/test/Fibonaqi.java b/Leecode/src/main/java/com/markilue/leecode/test/Fibonaqi.java index e9cfa0e..1de7bb5 100644 --- a/Leecode/src/main/java/com/markilue/leecode/test/Fibonaqi.java +++ b/Leecode/src/main/java/com/markilue/leecode/test/Fibonaqi.java @@ -1,38 +1,35 @@ package com.markilue.leecode.test; import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Scanner; public class Fibonaqi { - /** * 测试使用时间复杂度为n的斐波那契数列递归法 - * */ // @Test // public static void testFibonaqi(){ // // } - public static void main(String[] args) { - int n=5; - System.out.println(fibonacci(1,1,10)); + int n = 5; + System.out.println(fibonacci(1, 1, 10)); + Arrays.copyOf() } - public static int fibonacci(int first,int second,int n){ - if(n<=0){ + public static int fibonacci(int first, int second, int n) { + if (n <= 0) { return 0; } - if(n <3){ + if (n < 3) { return 1; - }else if(n==3){ - return first+second; - } - else { - return fibonacci(second,first+second,n-1); + } else if (n == 3) { + return first + second; + } else { + return fibonacci(second, first + second, n - 1); } } - - } diff --git a/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/ParaRCNN.py b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/ParaRCNN.py new file mode 100644 index 0000000..54a0e0e --- /dev/null +++ b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/ParaRCNN.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# -*- coding: UTF-8 -*- + + +import os +import sys +import pandas as pd +from math import sqrt +import matplotlib.pyplot as plt +import tensorflow as tf +from tensorflow import keras +from tensorflow.keras import Model, Input, layers +from tensorflow.keras.models import load_model +from tensorflow.keras.optimizers import Adam +from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint +from tensorflow.keras.layers import LSTM, SimpleRNN, GRU +from tensorflow.keras.layers import Dense, Dropout, concatenate + +from sklearn.metrics import mean_squared_error as mse +from sklearn.metrics import mean_absolute_error as mae + + +# adding data_processing to the system path +sys.path.insert(0, '/aul/homes/jshi008/IvyProjects/SimDL/data_processing') +from preprocessing import ws_preprocessing, pm25_preprocessing, ele_preprocessing + + + +''' +参考： +[1] https://arxiv.org/pdf/2305.04876v3.pdf +[2] https://github.com/JimengShi/ParaRCNN-Time-Series-Forecasting/blob/main/models/ParaRCNN_shifting_sc.py +''' + + +# set gpu +os.environ["CUDA_VISIBLE_DEVICES"]="3" + + +# get the preprocessed train_X, train_y, test_X, test_y from the dataset +n_hours = 72 +K = 24 +S = 24 + + +### pick one of datasets +# train_X, train_y, test_X, test_y, scaler = ws_preprocessing(n_hours, K, S) +# train_X, train_y, test_X, test_y, scaler = pm25_preprocessing(n_hours, K) +train_X, train_y, test_X, test_y, scaler = ele_preprocessing(n_hours, K) + + +# build model +model_input = Input(shape=(train_X.shape[1], train_X.shape[2])) + +### 4 cnn skip connection +x1 = layers.Conv1D(filters=256, + kernel_size=2, + activation='relu', + padding='same', + kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(model_input) + + +x1_merge1 = concatenate([model_input, x1]) +x1 = layers.Conv1D(filters=128, + kernel_size=2, + activation='relu', + padding='same', + kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x1_merge1) + + +x1_merge2 = concatenate([model_input, x1]) +x1 = layers.Conv1D(filters=64, + kernel_size=2, + activation='relu', + padding='same', + kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x1_merge2) + + +x1_merge3 = concatenate([model_input, x1]) +x1 = layers.Conv1D(filters=32, + kernel_size=2, + activation='relu', + padding='same', + kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x1_merge3) + + + +### 4 rnn skip connection +x2 = layers.SimpleRNN(128, + activation='relu', + return_sequences=True, + recurrent_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5), + kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(model_input) + + +x2_merge1 = concatenate([model_input, x2]) +x2 = layers.SimpleRNN(64, + activation='relu', + return_sequences=True, + recurrent_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5), + kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x2_merge1) + + +x2_merge2 = concatenate([model_input, x2]) +x2 = layers.SimpleRNN(32, + activation='relu', + return_sequences=True, + recurrent_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5), + kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x2_merge2) + + +x2_merge3 = concatenate([model_input, x2]) +x2 = layers.SimpleRNN(16, + activation='relu', + return_sequences=True, + recurrent_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5), + kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x2_merge3) + + +merge = concatenate([model_input, x1, x2]) +x = layers.Flatten()(merge) + +### uncomment the 3 rows below for pm25 and energy electricity dataset, comment them for water stage dataset +x = Dense(256, activation='relu', kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x) +x = Dense(128, activation='relu', kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x) +x = Dense(128, activation='relu', kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x) + + +x = Dense(train_y.shape[1])(x) + +model = Model(model_input, x) +model.summary() + + +# training +lr = 0.00001 +EPOCHS = 8000 + +lr_schedule = keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=5e-4, + decay_steps=10000, + decay_rate=0.99) + +model.compile( +# optimizer=Adam(learning_rate=lr, decay=lr/EPOCHS), + optimizer=Adam(learning_rate=lr_schedule), + loss='mse', + metrics=['mae']) + + +es = EarlyStopping(monitor='val_loss', mode='min', verbose=2, patience=1500) +mc = ModelCheckpoint('saved_model/ParaRCNN_shifting_s{}_k{}_sc.h5'.format(S, K), monitor='val_mae', mode='min', verbose=2, save_best_only=True) + +history = model.fit(train_X, train_y, + batch_size=512, + epochs=EPOCHS, + validation_data=(test_X, test_y), + verbose=2, + shuffle=True, + callbacks=[es, mc]) + +plt.rcParams["figure.figsize"] = (8, 6) +plt.plot(history.history['loss'], label='train') +plt.plot(history.history['val_loss'], label='test') +plt.xticks(fontsize=14) +plt.yticks(fontsize=14) +plt.xlabel('Epoch', fontsize=16) +plt.ylabel('Loss', fontsize=16) +plt.legend(fontsize=14) +plt.title("Training loss vs Testing loss", fontsize=18) +# plt.savefig('graph/rnn_loss.png', dpi=300) +plt.show() + + + +# Performance on test data +saved_model = load_model('saved_model/ParaRCNN_shifting_s{}_k{}_sc.h5'.format(S, K)) +yhat = saved_model.predict(test_X) + + +inv_yhat = scaler.inverse_transform(yhat) +inv_y = scaler.inverse_transform(test_y) + +inv_yhat = pd.DataFrame(inv_yhat) +inv_y = pd.DataFrame(inv_y) + + +print('MAE = {}'.format(float("{:.4f}".format(mae(inv_yhat.iloc[:, :], inv_y.iloc[:, :]))))) +print('RMSE = {}'.format(float("{:.4f}".format(sqrt(mse(inv_yhat.iloc[:, :], inv_y.iloc[:, :])))))) \ No newline at end of file diff --git a/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/__init__.py b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/attn.py b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/attn.py new file mode 100644 index 0000000..f8283c0 --- /dev/null +++ b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/attn.py @@ -0,0 +1,320 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import numpy as np + +from math import sqrt +from utils.masking import TriangularCausalMask, ProbMask + +''' +参考： +[1]https://arxiv.org/pdf/2307.00493v1.pdf +[2]https://github.com/nhatthanhtran/FWin2023/blob/main/models/model.py +''' +class FullAttention(nn.Module): + def __init__(self, mask_flag=True, factor=5, scale=None, attention_dropout=0.1, output_attention=False): + super(FullAttention, self).__init__() + self.scale = scale + self.mask_flag = mask_flag + self.output_attention = output_attention + self.dropout = nn.Dropout(attention_dropout) + + def forward(self, queries, keys, values, attn_mask): + B, L, H, E = queries.shape + _, S, _, D = values.shape + scale = self.scale or 1./sqrt(E) + + scores = torch.einsum("blhe,bshe->bhls", queries, keys) + if self.mask_flag: + if attn_mask is None: + attn_mask = TriangularCausalMask(B, L, device=queries.device) + + scores.masked_fill_(attn_mask.mask, -np.inf) + + A = self.dropout(torch.softmax(scale * scores, dim=-1)) + V = torch.einsum("bhls,bshd->blhd", A, values) + + if self.output_attention: + return (V.contiguous(), A) + else: + return (V.contiguous(), None) + + +class ProbAttention(nn.Module): + def __init__(self, mask_flag=True, factor=5, scale=None, attention_dropout=0.1, output_attention=False): + super(ProbAttention, self).__init__() + self.factor = factor + self.scale = scale + self.mask_flag = mask_flag + self.output_attention = output_attention + self.dropout = nn.Dropout(attention_dropout) + + def _prob_QK(self, Q, K, sample_k, n_top): # n_top: c*ln(L_q) + # Q [B, H, L, D] + B, H, L_K, E = K.shape + _, _, L_Q, _ = Q.shape + + # calculate the sampled Q_K + K_expand = K.unsqueeze(-3).expand(B, H, L_Q, L_K, E) + + + + # real U = U_part(factor*ln(L_k))*L_q + index_sample = torch.randint(L_K, (L_Q, sample_k)) + # print(index_sample.shape) + K_sample = K_expand[:, :, torch.arange( + L_Q).unsqueeze(1), index_sample, :] + Q_K_sample = torch.matmul( + Q.unsqueeze(-2), K_sample.transpose(-2, -1)).squeeze(-2) + + # find the Top_k query with sparisty measurement + M = Q_K_sample.max(-1)[0] - torch.div(Q_K_sample.sum(-1), L_K) + M_top = M.topk(n_top, sorted=False)[1] + + # use the reduced Q to calculate Q_K + Q_reduce = Q[torch.arange(B)[:, None, None], + torch.arange(H)[None, :, None], + M_top, :] # factor*ln(L_q) + + Q_K = torch.matmul(Q_reduce, K.transpose(-2, -1)) # factor*ln(L_q)*L_k + + return Q_K, M_top + + def _get_initial_context(self, V, L_Q): + B, H, L_V, D = V.shape + if not self.mask_flag: + # V_sum = V.sum(dim=-2) + V_sum = V.mean(dim=-2) + contex = V_sum.unsqueeze(-2).expand(B, H, + L_Q, V_sum.shape[-1]).clone() + else: # use mask + # requires that L_Q == L_V, i.e. for self-attention only + assert(L_Q == L_V) + contex = V.cumsum(dim=-2) + return contex + + def _update_context(self, context_in, V, scores, index, L_Q, attn_mask): + B, H, L_V, D = V.shape + + if self.mask_flag: + attn_mask = ProbMask(B, H, L_Q, index, scores, device=V.device) + scores.masked_fill_(attn_mask.mask, -np.inf) + + attn = torch.softmax(scores, dim=-1) # nn.Softmax(dim=-1)(scores) + + context_in[torch.arange(B)[:, None, None], + torch.arange(H)[None, :, None], + index, :] = torch.matmul(attn, V).type_as(context_in) + if self.output_attention: + attns = (torch.ones([B, H, L_V, L_V]) / + L_V).type_as(attn).to(attn.device) + attns[torch.arange(B)[:, None, None], torch.arange(H)[ + None, :, None], index, :] = attn + return (context_in, attns) + else: + return (context_in, None) + + def forward(self, queries, keys, values, attn_mask): + B, L_Q, H, D = queries.shape + _, L_K, _, _ = keys.shape + + queries = queries.transpose(2, 1) + keys = keys.transpose(2, 1) + values = values.transpose(2, 1) + + U_part = self.factor * \ + np.ceil(np.log(L_K)).astype('int').item() # c*ln(L_k) + u = self.factor * \ + np.ceil(np.log(L_Q)).astype('int').item() # c*ln(L_q) + + U_part = U_part if U_part < L_K else L_K + u = u if u < L_Q else L_Q + + scores_top, index = self._prob_QK( + queries, keys, sample_k=U_part, n_top=u) + + # add scale factor + scale = self.scale or 1./sqrt(D) + if scale is not None: + scores_top = scores_top * scale + # get the context + context = self._get_initial_context(values, L_Q) + + # update the context with selected top_k queries + context, attn = self._update_context( + context, values, scores_top, index, L_Q, attn_mask) + + return context.transpose(2, 1).contiguous(), attn + + +class AttentionLayer(nn.Module): + def __init__(self, attention, d_model, n_heads, + d_keys=None, d_values=None, mix=False, layer_num=0): + super(AttentionLayer, self).__init__() + + d_keys = d_keys or (d_model//n_heads) + d_values = d_values or (d_model//n_heads) + + self.inner_attention = attention + self.query_projection = nn.Linear(d_model, d_keys * n_heads) + self.key_projection = nn.Linear(d_model, d_keys * n_heads) + self.value_projection = nn.Linear(d_model, d_values * n_heads) + self.out_projection = nn.Linear(d_values * n_heads, d_model) + self.n_heads = n_heads + self.mix = mix + self.layer_num = layer_num + def forward(self, queries, keys, values, attn_mask): + + B, L, _ = queries.shape + _, S, _ = keys.shape + H = self.n_heads + + queries = self.query_projection(queries).view(B, L, H, -1) + keys = self.key_projection(keys).view(B, S, H, -1) + values = self.value_projection(values).view(B, S, H, -1) + + + out, attn = self.inner_attention( + queries, + keys, + values, + attn_mask + ) + + if self.mix: + out = out.transpose(2, 1).contiguous() + out = out.view(B, L, -1) + + return self.out_projection(out), attn + +class AttentionLayerWin(nn.Module): + def __init__(self, attention, d_model, n_heads, + d_keys=None, d_values=None, mix=False, layer_num=0, + window_size=8, output_attention=False): + super(AttentionLayerWin, self).__init__() + d_keys = d_keys or (d_model//n_heads) + d_values = d_values or (d_model//n_heads) + + self.inner_attention = attention + self.query_projection = nn.Linear(d_model, d_keys * n_heads) + self.key_projection = nn.Linear(d_model, d_keys * n_heads) + self.value_projection = nn.Linear(d_model, d_values * n_heads) + self.out_projection = nn.Linear(d_values * n_heads, d_model) + + self.n_heads = n_heads + self.mix = mix + self.layer_num = layer_num + self.window_size = window_size + self.output_attn = output_attention + def forward(self, queries, keys, values, attn_mask): + + B, L, _ = queries.shape + _, S, _ = keys.shape + H = self.n_heads + + queries = self.query_projection(queries).view(B, L, H, -1) + keys = self.key_projection(keys).view(B, S, H, -1) + values = self.value_projection(values).view(B, S, H, -1) + + #Partition the vectors into windows + queries = queries.view(B*(L//self.window_size), self.window_size, H, -1) + keys = keys.view(B*(S//self.window_size), self.window_size, H, -1) + values = values.view(B*(S//self.window_size), self.window_size, H, -1) + + + out, attn = self.inner_attention( + queries, + keys, + values, + attn_mask + ) + + if self.output_attn: + attn = self._output_attn(L, attn) + + out = out.view(B, L, H, -1) + + if self.mix: + out = out.transpose(2, 1).contiguous() + out = out.view(B, L, -1) + + return self.out_projection(out), attn + + def _output_attn(self,L, attn): + num_window = L//self.window_size + + for k in range(num_window): + if k==0: + p2d = (0,((num_window-(k+1))*self.window_size)) + attn_tmp = F.pad(attn[:self.window_size,:,:,:],p2d) + else: + p2d = (k*self.window_size, (num_window-(k+1))*self.window_size) + attn_tmp = torch.cat((attn_tmp, F.pad(attn[k*self.window_size:(k+1)*self.window_size,:,:,:],p2d)),dim=2) + + return attn_tmp + + +class AttentionLayerCrossWin(nn.Module): + def __init__(self, attention, d_model, n_heads, + d_keys=None, d_values=None, mix=False, layer_num=0, + num_windows=4, output_attention=False): + super(AttentionLayerCrossWin, self).__init__() + d_keys = d_keys or (d_model//n_heads) + d_values = d_values or (d_model//n_heads) + + self.inner_attention = attention + self.query_projection = nn.Linear(d_model, d_keys * n_heads) + self.key_projection = nn.Linear(d_model, d_keys * n_heads) + self.value_projection = nn.Linear(d_model, d_values * n_heads) + self.out_projection = nn.Linear(d_values * n_heads, d_model) + + self.n_heads = n_heads + self.mix = mix + self.layer_num = layer_num + self.num_windows = num_windows + self.output_attn = output_attention + def forward(self, queries, keys, values, attn_mask): + + B, L, _ = queries.shape + _, S, _ = keys.shape + H = self.n_heads + + queries = self.query_projection(queries).view(B, L, H, -1) + keys = self.key_projection(keys).view(B, S, H, -1) + values = self.value_projection(values).view(B, S, H, -1) + + #Partition the vectors into windows + queries = queries.view(B*self.num_windows, L//self.num_windows, H, -1) + keys = keys.view(B*self.num_windows, S//self.num_windows, H, -1) + values = values.view(B*self.num_windows, S//self.num_windows, H, -1) + + out, attn = self.inner_attention( + queries, + keys, + values, + attn_mask + ) + + if self.output_attn: + attn = self._output_attn(L, attn) + + out = out.view(B, L, H, -1) + + if self.mix: + out = out.transpose(2, 1).contiguous() + out = out.view(B, L, -1) + + return self.out_projection(out), attn + + def _output_attn(self,L, attn): + window_size = L//self.num_windows + + for k in range(self.num_window): + if k==0: + p2d = (0,((self.num_windows-(k+1))*window_size)) + attn_tmp = F.pad(attn[:window_size,:,:,:],p2d) + else: + p2d = (k*window_size, (self.num_windows-(k+1))*window_size) + attn_tmp = torch.cat((attn_tmp, F.pad(attn[k*window_size:(k+1)*window_size,:,:,:],p2d)),dim=2) + + return attn_tmp \ No newline at end of file diff --git a/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/attn_tensorflow.py b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/attn_tensorflow.py new file mode 100644 index 0000000..519d07c --- /dev/null +++ b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/attn_tensorflow.py @@ -0,0 +1,130 @@ +import tensorflow as tf +from tensorflow.keras import layers + +class AttentionLayerWin(tf.keras.layers.Layer): + def __init__(self, attention, d_model, n_heads, + d_keys=None, d_values=None, mix=False, layer_num=0, + window_size=8, output_attention=False): + super(AttentionLayerWin, self).__init__() + d_keys = d_keys or (d_model // n_heads) + d_values = d_values or (d_model // n_heads) + + self.inner_attention = attention + self.query_projection = layers.Dense(d_keys * n_heads) + self.key_projection = layers.Dense(d_keys * n_heads) + self.value_projection = layers.Dense(d_values * n_heads) + self.out_projection = layers.Dense(d_model) + + self.n_heads = n_heads + self.mix = mix + self.layer_num = layer_num + self.window_size = window_size + self.output_attn = output_attention + + def call(self, queries, keys, values, attn_mask): + B, L, _ = queries.shape + _, S, _ = keys.shape + H = self.n_heads + + queries = tf.reshape(self.query_projection(queries), (B, L, H, -1)) + keys = tf.reshape(self.key_projection(keys), (B, S, H, -1)) + values = tf.reshape(self.value_projection(values), (B, S, H, -1)) + + queries = tf.reshape(queries, (B * (L // self.window_size), self.window_size, H, -1)) + keys = tf.reshape(keys, (B * (S // self.window_size), self.window_size, H, -1)) + values = tf.reshape(values, (B * (S // self.window_size), self.window_size, H, -1)) + + out, attn = self.inner_attention( + queries, + keys, + values, + attn_mask + ) + + if self.output_attn: + attn = self._output_attn(L, attn) + + out = tf.reshape(out, (B, L, H, -1)) + + if self.mix: + out = tf.transpose(out, (0, 2, 1, 3)) + out = tf.reshape(out, (B, L, -1)) + + return self.out_projection(out), attn + + def _output_attn(self, L, attn): + num_window = L // self.window_size + + for k in range(num_window): + if k == 0: + p2d = ((0, ((num_window - (k + 1)) * self.window_size)), (0, 0)) + attn_tmp = tf.pad(attn[:self.window_size, :, :, :], p2d) + else: + p2d = ((k * self.window_size, (num_window - (k + 1)) * self.window_size), (0, 0)) + attn_tmp = tf.concat((attn_tmp, tf.pad(attn[k * self.window_size:(k + 1) * self.window_size, :, :, :], p2d)), axis=2) + + return attn_tmp + +class AttentionLayerCrossWin(tf.keras.layers.Layer): + def __init__(self, attention, d_model, n_heads, + d_keys=None, d_values=None, mix=False, layer_num=0, + num_windows=4, output_attention=False): + super(AttentionLayerCrossWin, self).__init__() + d_keys = d_keys or (d_model // n_heads) + d_values = d_values or (d_model // n_heads) + + self.inner_attention = attention + self.query_projection = layers.Dense(d_keys * n_heads) + self.key_projection = layers.Dense(d_keys * n_heads) + self.value_projection = layers.Dense(d_values * n_heads) + self.out_projection = layers.Dense(d_model) + + self.n_heads = n_heads + self.mix = mix + self.layer_num = layer_num + self.num_windows = num_windows + self.output_attn = output_attention + + def call(self, queries, keys, values, attn_mask): + B, L, _ = queries.shape + _, S, _ = keys.shape + H = self.n_heads + + queries = tf.reshape(self.query_projection(queries), (B, L, H, -1)) + keys = tf.reshape(self.key_projection(keys), (B, S, H, -1)) + values = tf.reshape(self.value_projection(values), (B, S, H, -1)) + + queries = tf.reshape(queries, (B * self.num_windows, L // self.num_windows, H, -1)) + keys = tf.reshape(keys, (B * self.num_windows, S // self.num_windows, H, -1)) + values = tf.reshape(values, (B * self.num_windows, S // self.num_windows, H, -1)) + + out, attn = self.inner_attention( + queries, + keys, + values, + attn_mask + ) + + if self.output_attn: + attn = self._output_attn(L, attn) + + out = tf.reshape(out, (B, L, H, -1)) + + if self.mix: + out = tf.transpose(out, (0, 2, 1, 3)) + out = tf.reshape(out, (B, L, -1)) + + return self.out_projection(out), attn + + def _output_attn(self, L, attn): + window_size = L // self.num_windows + + for k in range(self.num_windows): + if k == 0: + p2d = ((0, ((self.num_windows - (k + 1)) * window_size)), (0, 0)) + attn_tmp = tf.pad(attn[:window_size, :, :, :], p2d) + else: + p2d = ((k * window_size, (self.num_windows - (k + 1)) * window_size), (0, 0)) + attn_tmp = tf.concat((attn_tmp, tf.pad(attn[k * window_size:(k + 1) * window_size, :, :, :], p2d)), axis=2) + + return attn_tmp diff --git a/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/fourier.py b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/fourier.py new file mode 100644 index 0000000..110821b --- /dev/null +++ b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/fourier.py @@ -0,0 +1,71 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +import numpy as np + +''' +参考： +[1]https://arxiv.org/pdf/2307.00493v1.pdf +[2]https://github.com/nhatthanhtran/FWin2023/blob/main/models/model.py +''' + +class FourierMix(nn.Module): + def __init__(self, d_model): + super().__init__() + self.fourier = FourierLayer(1, 2) + self.norm = nn.LayerNorm(d_model) + + def forward(self, x, attn_mask=None): + x = self.fourier(x) + x = self.norm(x) + return x, None + + +class FourierLayer(nn.Module): + def __init__(self, seq_dim=1, hidden_dim=2): + super().__init__() + self.seq_dim = seq_dim + self.hidden_dim = hidden_dim + + def forward(self, x): + B, L, D = x.shape + return torch.real(torch.fft.fft(torch.fft.fft(x, dim=self.hidden_dim), dim=self.seq_dim)) + + +class FeedFoward(nn.Module): + def __init__(self, d_model, dropout=0.1, activation="relu") -> None: + super().__init__() + self.fc1 = nn.Linear(d_model, d_model) + self.activation = F.relu if activation == "relu" else F.gelu + self.fc2 = nn.Linear(d_model, d_model) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + x = self.fc1(x) + x = self.dropout(x) + x = self.activation(x) + x = self.dropout(self.fc2(x)) + + return x + + +class FNetLayer(nn.Module): + def __init__(self, d_model, dropout=0.1, activation="relu"): + super().__init__() + self.fourier = FourierLayer(1, 2) + self.feedforward = FeedFoward(d_model, dropout, activation) + self.dropout = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + + def forward(self, x, attn_mask=None): + new_x = self.fourier(x) + x = x + self.dropout(new_x) + + x = self.norm1(x) + x = x + self.feedforward(x) + return self.norm2(x), None + + + diff --git a/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/fourier_tensorflow.py b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/fourier_tensorflow.py new file mode 100644 index 0000000..3fa5afe --- /dev/null +++ b/TensorFlow_eaxmple/Model_train_test/RUL/otherIdea/fourier_tensorflow.py @@ -0,0 +1,57 @@ +import tensorflow as tf +from tensorflow.keras import layers + +class FourierLayer(tf.keras.layers.Layer): + def __init__(self, seq_dim=1, hidden_dim=2): + super(FourierLayer, self).__init__() + self.seq_dim = seq_dim + self.hidden_dim = hidden_dim + + def call(self, x): + B, L, D = x.shape + return tf.math.real( + tf.signal.fft(tf.signal.fft(x, axis=self.hidden_dim), axis=self.seq_dim) + ) + +class FeedForward(tf.keras.layers.Layer): + def __init__(self, d_model, dropout=0.1, activation="relu"): + super(FeedForward, self).__init__() + self.fc1 = layers.Dense(d_model) + self.activation = layers.ReLU() if activation == "relu" else layers.GELU() + self.fc2 = layers.Dense(d_model) + self.dropout = layers.Dropout(dropout) + + def call(self, x): + x = self.fc1(x) + x = self.dropout(x) + x = self.activation(x) + x = self.dropout(self.fc2(x)) + return x + +class FNetLayer(tf.keras.layers.Layer): + def __init__(self, d_model, dropout=0.1, activation="relu"): + super(FNetLayer, self).__init__() + self.fourier = FourierLayer(seq_dim=1, hidden_dim=2) + self.feedforward = FeedForward(d_model, dropout, activation) + self.dropout = layers.Dropout(dropout) + self.norm1 = layers.LayerNormalization() + self.norm2 = layers.LayerNormalization() + + def call(self, x, attn_mask=None): + new_x = self.fourier(x) + x = x + self.dropout(new_x) + + x = self.norm1(x) + x = x + self.feedforward(x) + return self.norm2(x), None + +class FourierMix(tf.keras.layers.Layer): + def __init__(self, d_model): + super(FourierMix, self).__init__() + self.fourier = FourierLayer(seq_dim=1, hidden_dim=2) + self.norm = layers.LayerNormalization() + + def call(self, x, attn_mask=None): + x = self.fourier(x) + x = self.norm(x) + return x, None