论文方面更新

GE_Migrating_data/src/main/java/com/cqu/ge/DataServiceImplTest.java

@@ -185,6 +185,7 @@ public class DataServiceImplTest {
         assertEquals(tagName1, result.get(0).getTagName());
         assertEquals(tagName2, result.get(1).getTagName());
 
+
 //		if (result.get(0).getTagName().equals(tagName1) && result.get(1).getTagName().equals(tagName2)) {
 //			DataSample[] datasamples1 = result.get(0).getSamples();
 //			DataSample[] datasamples2 = result.get(1).getSamples();

Spider/Chapter10_模拟登录/大规模账号池的搭建/AccountPool/accountpool/processors/generator.py

@@ -14,7 +14,8 @@ class BaseGenerator(object):
             raise InitException
         self.account_operator = RedisClient(type='account', website=self.website)
         self.credential_operator = RedisClient(type='credential', website=self.website)
-    
+
+    # 所有子类重写generate方法,写自己的生成逻辑
     def generate(self, username, password):
         """
         generate method
@@ -22,14 +23,16 @@ class BaseGenerator(object):
         :param password: password
         :return:
         """
+        # 这里直接抛出了NotImplementedError异常，因此子类必须实现该方法，否则运行时会报错
         raise NotImplementedError
-    
+
     def init(self):
         """
         do init
         """
         pass
-    
+
+    # 模板代码设计模式，写标准的run流程
     def run(self):
         """
         run main process
@@ -48,7 +51,7 @@ import requests
 
 
 class Antispider6Generator(BaseGenerator):
-    
+
     def init(self):
         """
         do init
@@ -56,7 +59,7 @@ class Antispider6Generator(BaseGenerator):
         if self.account_operator.count() == 0:
             self.account_operator.set('admin', 'admin')
             self.account_operator.set('admin2', 'admin2')
-    
+
     def generate(self, username, password):
         """
         generate main process
@@ -66,6 +69,7 @@ class Antispider6Generator(BaseGenerator):
             return
         login_url = 'https://antispider6.scrape.center/login'
         s = requests.Session()
+        # 这里省去了注册账号的过程
         s.post(login_url, data={
             'username': username,
             'password': password
@@ -80,7 +84,6 @@ class Antispider6Generator(BaseGenerator):
 
 
 class Antispider7Generator(BaseGenerator):
-
     MAX_COUNT = 100
 
     def init(self):

Spider/Chapter10_模拟登录/大规模账号池的搭建/AccountPool/accountpool/processors/server.py

@@ -24,6 +24,7 @@ def get_conn():
     """
     for website in GENERATOR_MAP:
         if not hasattr(g, website):
+            # 这里的get和set本质上可以看成建立一个map
             setattr(g, f'{website}_{credential}', RedisClient(credential, website))
             setattr(g, f'{website}_{account}', RedisClient(account, website))
     return g

Spider/Chapter10_模拟登录/大规模账号池的搭建/AccountPool/accountpool/storages/redis.py

@@ -10,6 +10,8 @@ class RedisClient(object):
     
     def __init__(self, type, website, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD):
         """
+        type: 是存储的account信息；还是Cookie相关信息
+        website: 爬取的对应网站
         init redis client
         :param host: redis host
         :param port: redis port
@@ -18,9 +20,11 @@ class RedisClient(object):
         self.db = redis.StrictRedis(host=host, port=port, password=password, decode_responses=True)
         self.type = type
         self.website = website
-    
+
+    # 这里的name相当于redis-key
     def name(self):
         """
+        这里是做二级分类，二级hash，方便存取
         get hash name
         :return: name of hash
         """

Spider/Chapter10_模拟登录/大规模账号池的搭建/AccountPool/importer.py

@@ -1,6 +1,7 @@
 from accountpool.storages.redis import RedisClient
 import argparse
 
+# 将生成的账号导入账号池
 parser = argparse.ArgumentParser(description='AccountPool')
 parser.add_argument('website', type=str, help='website')
 args = parser.parse_args()

Spider/Chapter10_模拟登录/大规模账号池的搭建/AccountPool/register.py

@@ -3,6 +3,7 @@ from acinonyx import run
 import requests
 from loguru import logger
 
+# 具体用法参考：https://pypi.org/project/acinonyx/
 # This is a script for registering account for antispider7, using acinonyx to accelerate.
 
 parser = argparse.ArgumentParser(description='AccountPool')
@@ -23,6 +24,7 @@ def register(username, password):
 
 if __name__ == '__main__':
     accounts = []
+    # 注册
     for index in range(1, 1000):
         accounts.append((f'admin{index}', f'admin{index}'))
     run(register, accounts)

TensorFlow_eaxmple/Model_train_test/2012轴承数据集预测挑战/HI_create/HI_merge.py

@@ -55,28 +55,28 @@ class model():
 
 
 # HI指标训练和合成
-if __name__ == '__main__':
+# if __name__ == '__main__':
-    model = model(input_shape=6).getModel(model_Type='ae')
+#     model = model(input_shape=6).getModel(model_Type='ae')
-    model.compile(optimizer=tf.optimizers.Adam(0.001), loss=tf.losses.mse, metrics=['acc'])
+#     model.compile(optimizer=tf.optimizers.Adam(0.001), loss=tf.losses.mse, metrics=['acc'])
-    # model.summary()
+#     # model.summary()
-
+#
-    checkpoint = tf.keras.callbacks.ModelCheckpoint(
+#     checkpoint = tf.keras.callbacks.ModelCheckpoint(
-        filepath="AE_model.h5",
+#         filepath="AE_model.h5",
-        monitor='val_loss',
+#         monitor='val_loss',
-        verbose=2,
+#         verbose=2,
-        save_best_only=True,
+#         save_best_only=True,
-        mode='min')
+#         mode='min')
-    lr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, min_lr=0.0001)
+#     lr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, min_lr=0.0001)
-
+#
-    model.compile(optimizer=tf.optimizers.SGD(), loss=tf.losses.mse)
+#     model.compile(optimizer=tf.optimizers.SGD(), loss=tf.losses.mse)
-    # model.compile(optimizer=tf.optimizers.SGD(learning_rate=0.001), loss=FTMSE())
+#     # model.compile(optimizer=tf.optimizers.SGD(learning_rate=0.001), loss=FTMSE())
-    model.summary()
+#     model.summary()
-    early_stop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20, mode='min', verbose=1)
+#     early_stop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20, mode='min', verbose=1)
-
+#
-    history = model.fit(train_data, train_data, epochs=1000, batch_size=100)
+#     history = model.fit(train_data, train_data, epochs=1000, batch_size=100)
-    HI_merge_data = tf.keras.models.Model(inputs=model.input, outputs=model.get_layer('mid').output).predict(train_data)
+#     HI_merge_data = tf.keras.models.Model(inputs=model.input, outputs=model.get_layer('mid').output).predict(train_data)
-    print(HI_merge_data)
+#     print(HI_merge_data)
-    acc = np.array(history.history.get('acc'))
+#     acc = np.array(history.history.get('acc'))
-    # if acc[299] > 0.9:
+#     # if acc[299] > 0.9:
-    np.save("HI_merge_data1.npy", HI_merge_data)
+#     np.save("HI_merge_data1.npy", HI_merge_data)
-    model.save("AE_model.h5")
+#     model.save("AE_model.h5")

TensorFlow_eaxmple/Model_train_test/RUL/FFTUtils.py

@@ -48,8 +48,8 @@ if __name__ == '__main__':
                       0.343256533146,
                       0.008932195604])
 
-    print(tf.signal.fft(array))
+    print(np.fft.fft(array))
     array_fft = np.fft.fft(array)
 
-    # print(ifft(array))
+    print(ifft(array))
-    print(array_fft)
+    # print(array_fft)

TensorFlow_eaxmple/Model_train_test/RUL/test/FFTTest.py

@@ -40,16 +40,16 @@ angle1 = np.arctan2(imag, real)
 
 print(angle)
 print(angle1)
-# result = []
+result = []
-# (num,) = amp.shape
+(num,) = amp.shape
-# (total,) = array.shape
+(total,) = array.shape
-# for j in range(total):
+for j in range(total):
-#     wk = 2 * np.pi * np.arange(num) / num
+    wk = 2 * np.pi * np.arange(num) / num
-#     cur = np.sum(amp * np.cos(wk * j + angle))
+    cur = np.sum(amp * np.cos(wk * j + angle))
-#     result.append(cur)
+    result.append(cur)
-#
+
-# print(result)
+print(result)
-# print(array)
+print(array)
 # # print(array_fft)
 #
 # plt.subplot(2, 1, 1)

TensorFlow_eaxmple/Model_train_test/RUL/test/test1.py

@@ -0,0 +1,33 @@
+#-*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2024/1/11 14:20
+@Usage : 
+@Desc :
+'''
+
+import requests
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+def getData():
+    url = 'http://172.28.9.61:8100/api/data/signalData?dataset_name=Shaft2_radial&thing_id=windturbine-rmq-01&time=20240110200508000'
+
+    response = requests.get(url)
+
+    y = response.json()['data']['timeSequence_y']
+
+    y = np.array(y)
+    amp = np.abs(np.fft.fft(y) / len(y))
+
+    plt.plot(amp[:len(amp)//2])
+    plt.show()
+
+    print(1)
+
+
+
+if __name__ == '__main__':
+    getData()

TensorFlow_eaxmple/Model_train_test/condition_monitoring/healthyScore.py

@@ -0,0 +1,85 @@
+# -*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2023/12/22 19:42
+@Usage : 
+@Desc : 测试计算健康度
+'''
+
+import matplotlib.pyplot as plt
+import math
+import numpy as np
+
+ucl = 4
+
+
+def func(x, ucl):
+    y = []
+    z = []
+    s = []
+    g = []
+    f = []
+    for i in x:
+        each_g = -math.exp(math.log(2) / ucl * i) + 2
+        each_f = 2 / (1 + math.exp((i - ucl) * 1.5)) - 1
+        if i < ucl:
+            each_z = (2 / (1 + math.exp(-(ucl / i - 1))) - 1)
+        else:
+            each_z = (2 / (1 + math.exp(((i - ucl) / (2 * ucl - i)))) - 1)
+        each_s = math.cos(math.pi * i / (2 * ucl))
+        each_y = 50 + 50 * each_z
+        y.append(each_y)
+        z.append(each_z)
+        s.append(each_s)
+        g.append(each_g)
+        f.append(each_f)
+    return y, z, s, g, f
+    pass
+
+
+def healthy_score(x):
+    weight = 2 / (1 + math.exp((x - ucl) * 1.5)) - 1
+    healthy_score = 60 + 40 * weight
+    return healthy_score
+
+
+x = np.linspace(0.1, 2 * ucl - 0.1, 40)
+y, z, s, g, f = func(x, 4)
+print(x)
+
+print(z)
+print(f)
+print(s)
+print(healthy_score(ucl))
+# plt.subplot(4, 1, 1)
+# plt.plot(x, y)
+# plt.subplot(4, 1, 2)
+# plt.plot(x, z)
+# plt.subplot(4, 1, 3)
+# plt.plot(x, f)
+# plt.subplot(4, 1, 4)
+# plt.plot(x, s)
+# # plt.subplot(4, 1, 4)
+# # plt.plot(x, g)
+# plt.show()
+plt.figure(2)
+plt.rc('font', family='Times New Roman')  # 全局字体样式
+
+plt.plot(x, f, label='a')
+plt.plot(x, z, label='b')
+plt.plot(x, s, label='c')
+font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 16}  # 设置坐标标签的字体大小，字体
+font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 13}  # 设置坐标标签的字体大小，字体
+indices = [i for i in range(8)]
+y_indices = [-1, -0.5, 0, 0.5, 1]
+x_classes = ['UCL/4', 'UCL/2', '3UCL/4', 'UCL', '5UCL/4', '3UCL/2', '7UCL/4', '2UCL']
+y_classes = ['0', '25', '50', '75', '100']
+plt.xticks([index + 0.5 for index in indices], x_classes, fontproperties=font1)  # 设置横坐标方向，rotation=45为45度倾斜
+plt.yticks([index for index in y_indices], y_classes, fontproperties=font1)  # 设置横坐标方向，rotation=45为45度倾斜
+# pad调整label与坐标轴之间的距离
+plt.tick_params(bottom=True, top=False, left=True, right=False, direction='inout', length=5, width=1, pad=1)
+plt.ylabel('Healthy Mapping', fontdict=font2)
+plt.xlabel('R$_t$', fontdict=font2)
+plt.legend(fontsize=16)
+plt.show()

TensorFlow_eaxmple/Model_train_test/condition_monitoring/others_idea/CNN_GRU.py

@@ -210,28 +210,29 @@ if __name__ == '__main__':
     print("===============================")
 
 
-    test_data, test_label = get_training_data(total_data[20000:, :])
+    # test_data, test_label = get_training_data(total_data[20000:, :])
-    predicted_data = newModel.predict(test_data)
+    # predicted_data = newModel.predict(test_data)
-    rows, cols = predicted_data.shape
+    # rows, cols = predicted_data.shape
-    print("=====================================")
+    # print("=====================================")
-    print(predicted_data)
+    # print(predicted_data)
-    print(predicted_data.shape)
+    # print(predicted_data.shape)
-    print("=====================================")
+    # print("=====================================")
 
-    temp = np.abs(predicted_data - test_label)
+    # temp = np.abs(predicted_data - test_label)
-    temp1 = (temp - np.broadcast_to(np.mean(temp, axis=0), shape=predicted_data.shape))
+    # temp1 = (temp - np.broadcast_to(np.mean(temp, axis=0), shape=predicted_data.shape))
-    temp2 = np.broadcast_to(np.sqrt(np.var(temp, axis=0)), shape=predicted_data.shape)
+    # temp2 = np.broadcast_to(np.sqrt(np.var(temp, axis=0)), shape=predicted_data.shape)
-    temp3 = temp1 / temp2
+    # temp3 = temp1 / temp2
-    mse = np.sum((temp1 / temp2) ** 2, axis=1)
+    # mse = np.sum((temp1 / temp2) ** 2, axis=1)
-    print("====================")
+    # print("====================")
-    print("new_mse:",mse)
+    # print("new_mse:",mse)
-    print(mse.shape)
+    # print(mse.shape)
-    np.savetxt("mse", mse, delimiter=',')
+    # np.savetxt("mse", mse, delimiter=',')
-    print("===================")
+    # print("===================")
 
-    plt.plot(mse[2000:])
+    # plt.plot(mse[2000:])
-    plt.plot(mean)
+    plt.plot(mse)
-    plt.plot(max)
+    # plt.plot(mean)
+    # plt.plot(max)
     plt.show()
 
 
@@ -239,24 +240,24 @@ if __name__ == '__main__':
 
 
 
-    data = pd.DataFrame(mse).ewm(span=3).mean()
+    # data = pd.DataFrame(mse).ewm(span=3).mean()
-    print(data)
+    # print(data)
-    data =np.array(data)
+    # data =np.array(data)
-
+    #
-    index,_ = data.shape
+    # index,_ = data.shape
-
+    #
-
+    #
-
+    #
-    for i in range(2396):
+    # for i in range(2396):
-        if data[i,0] >5:
+    #     if data[i,0] >5:
-            data[i,0] = data[i-1,:]
+    #         data[i,0] = data[i-1,:]
-    print(data)
+    # print(data)
-    mean = data[2000:2396,:].mean()
+    # mean = data[2000:2396,:].mean()
-    std = data[2000:2396,:].std()
+    # std = data[2000:2396,:].std()
-    mean=np.broadcast_to(mean,shape=[500,])
+    # mean=np.broadcast_to(mean,shape=[500,])
-    std=np.broadcast_to(std,shape=[500,])
+    # std=np.broadcast_to(std,shape=[500,])
-    plt.plot(data[2000:2396,:])
+    # plt.plot(data[2000:2396,:])
-    plt.plot(mean)
+    # plt.plot(mean)
-    plt.plot(mean+3*std)
+    # plt.plot(mean+3*std)
-    plt.plot(mean-3*std)
+    # plt.plot(mean-3*std)
-    plt.show()
+    # plt.show()

TensorFlow_eaxmple/Model_train_test/condition_monitoring/plot/test_plot.py

@@ -917,6 +917,7 @@ def plot_loss():
 
 
 if __name__ == '__main__':
+    # plot_mse_single()
     # model_list = ["DCConv", "GRU"]
     #
     # for model_name in model_list:
@@ -1030,8 +1031,8 @@ if __name__ == '__main__':
     # ]
     # # acc(list)
     #
-    list=[98.56,98.95,99.95,96.1,95,99.65,76.25,72.64,75.87,68.74,88.36,95.52]
+    # list=[98.56,98.95,99.95,96.1,95,99.65,76.25,72.64,75.87,68.74,88.36,95.52]
-    plot_FNR1(list)
+    # plot_FNR1(list)
     # # # #
     # # list=[3.43,1.99,1.92,2.17,1.63,1.81,1.78,1.8,0.6]
     # list=[3.43,0,0,0,0,0,1.8,0,0,0,0,0,0.6,0]
@@ -1048,7 +1049,7 @@ if __name__ == '__main__':
     # 单独预测图
     # plot_mse('E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\JM_banda/banda_joint_result_predict3.csv',data='E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\JM_banda/raw_data.csv')
     # plot_mse_single('E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\RNet_D/banda\RNet_D_banda_mse_predict1.csv')
-    # plot_mse_single('E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\RNet_3\RNet_3_timestamp120_feature10_result.csv')
+    plot_mse_single('E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\RNet_3\RNet_3_timestamp120_feature10_result.csv')
     # 画3d图
     # plot_3d()
 

TensorFlow_eaxmple/Model_train_test/wind_turbine_predict/HI_create/step1_Original_data.py

@@ -25,7 +25,6 @@ z = 0
 L.sort(key=lambda x:x.split(".")[1].split("_")[1])
 for filename in L:
     print("读取了{0}个文件".format(z+1))
-    # data_single = pd.read_csv(filename, dtype=np.float32, header=None)
     data_single = np.loadtxt(filename, delimiter=',',dtype=np.str)
     # data_single = data_single.iloc[0, :].values
     if z == 0:

pytorch_example/HealthyScorePredict/__init__.py

@@ -0,0 +1,8 @@
+#-*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2024/1/10 14:55
+@Usage : 
+@Desc :
+'''

pytorch_example/HealthyScorePredict/test/__init__.py

@@ -0,0 +1,8 @@
+#-*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2024/1/10 14:56
+@Usage : 
+@Desc :
+'''

pytorch_example/HealthyScorePredict/test/healthyScore.py

@@ -0,0 +1,58 @@
+# -*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2023/12/22 19:42
+@Usage : 
+@Desc : 测试计算健康度
+'''
+
+import matplotlib.pyplot as plt
+import math
+import numpy as np
+
+
+
+
+def func(x, ucl):
+    f = []
+    for i in x:
+        each_f = 2 / (1 + math.exp((i - ucl) * 1.5)) - 1
+        f.append(each_f)
+    return f
+    pass
+
+
+def cal_healthy_score(x,ucl):
+    healthy_score = []
+    for i in x:
+        weight = 2 / (1 + math.exp((i - ucl) * 1.5)) - 1
+        healthy_score.append(50 + 50 * weight)
+    return healthy_score
+
+
+if __name__ == '__main__':
+    ucl = 4
+    x = np.linspace(0.1, 2 * ucl - 0.1, 40)
+    y, z, s, g, f = func(x, 4)
+
+    plt.figure(2)
+    plt.rc('font', family='Times New Roman')  # 全局字体样式
+
+    plt.plot(x, f, label='a')
+    plt.plot(x, z, label='b')
+    plt.plot(x, s, label='c')
+    font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 16}  # 设置坐标标签的字体大小，字体
+    font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 13}  # 设置坐标标签的字体大小，字体
+    indices = [i for i in range(8)]
+    y_indices = [-1, -0.5, 0, 0.5, 1]
+    x_classes = ['UCL/4', 'UCL/2', '3UCL/4', 'UCL', '5UCL/4', '3UCL/2', '7UCL/4', '2UCL']
+    y_classes = ['0', '25', '50', '75', '100']
+    plt.xticks([index + 0.5 for index in indices], x_classes, fontproperties=font1)  # 设置横坐标方向，rotation=45为45度倾斜
+    plt.yticks([index for index in y_indices], y_classes, fontproperties=font1)  # 设置横坐标方向，rotation=45为45度倾斜
+    # pad调整label与坐标轴之间的距离
+    plt.tick_params(bottom=True, top=False, left=True, right=False, direction='inout', length=5, width=1, pad=1)
+    plt.ylabel('Healthy Mapping', fontdict=font2)
+    plt.xlabel('R$_t$', fontdict=font2)
+    plt.legend(fontsize=16)
+    plt.show()

pytorch_example/HealthyScorePredict/test/plotHealthyScore.py

@@ -0,0 +1,102 @@
+# -*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2024/1/10 14:56
+@Usage : 
+@Desc :
+'''
+import numpy as np
+import matplotlib.pyplot as plt
+from healthyScore import cal_healthy_score
+
+
+def calHealthyScore(data, scope):
+    mu = np.mean(data[:scope], axis=0)
+    sigma = np.std(data[:scope], axis=0)
+
+    healthy_score = cal_healthy_score(data, mu + 3 * sigma)
+
+    print(healthy_score)
+    # 画健康度图
+    font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 16}  # 设置坐标标签的字体大小，字体
+    font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 13}  # 设置坐标标签的字体大小，字体
+    plt.rc('font', family='Times New Roman')  # 全局字体样式
+    plt.ylabel('Healthy Score', fontdict=font2)
+    plt.xlabel('time', fontdict=font2)
+    # plt.legend(fontsize=16)
+    plt.plot(healthy_score)
+    plt.show()
+
+    return np.array(healthy_score)
+
+
+# LSTM_cell的数目,维度,是否正则化
+def getData(filter_num, dims, if_norm: bool = False):
+    # 数据读入
+    HI_merge_data_origin = np.load("../dataset/HI_merge_data.npy")
+    HI_merge_data = HI_merge_data_origin[0:1250, 1]
+    HI_merge_data = calHealthyScore(HI_merge_data, 800)
+    np.savetxt("../dataset/Healthy_score.csv",HI_merge_data,delimiter=',')
+    # plt.plot(HI_merge_data)
+    # plt.show()
+
+    # 去除掉退化特征不明显前面的点
+    # HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\RUL\dataset\smallVHI.csv", delimiter=",")
+
+    # # 是否正则化
+    # if if_norm:
+    #     HI_merge_data = normalization(HI_merge_data)
+    #
+    # # plt.plot(HI_merge_data)
+    # # plt.show()
+    # (total_dims,) = HI_merge_data.shape
+    #
+    # # # 将其分成重叠采样状态-滑动窗口函数
+    # predict_data = np.empty(shape=[total_dims - filter_num, filter_num])
+    #
+    # # 重叠采样获取时间部和训练次数
+    # for dim in range(total_dims - filter_num):
+    #     predict_data[dim] = HI_merge_data[dim:dim + filter_num]
+    #
+    # train_label = predict_data[dims:, :]
+    # train_label_single = HI_merge_data[dims + filter_num - 1:-1]
+    #
+    # # 再重叠采样获取一个点的维度
+    # '''train_data.shape:(sample,filter_num)  ->  (sample,filter_num,dims)'''
+    #
+    # # # 将其分成重叠采样状态-滑动窗口函数
+    # train_data = np.empty(shape=[dims, total_dims - filter_num - dims, filter_num])
+    #
+    # for dim in range(dims):
+    #     train_data[dim] = predict_data[dim:total_dims - filter_num - dims + dim, :]
+    #
+    # # 转置变成想要的数据 (dims,sample,filter_num)  ->  (sample,filter_num,dims)
+    #
+    # train_data = np.transpose(train_data, [1, 2, 0])
+    #
+    # # todo 解决模型保存时,query无法序列化的问题
+    # total_data = HI_merge_data
+    #
+    # print("total_data.shape:", total_data.shape)
+    # print("train_data.shape:", train_data.shape)  # (20, 1200, 30)
+    # print("train_label.shape:", train_label.shape)  # (20, 1200)
+    # print("train_label_single.shape:", train_label_single.shape)
+    #
+    # # 所有的原始数据;所有的训练数据;所有的训练标签(预测一个序列);所有的训练标签(预测一个点)
+    # return total_data, train_data, train_label, train_label_single
+
+
+def standardization(data):
+    mu = np.mean(data, axis=0)
+    sigma = np.std(data, axis=0)
+    return (data - mu) / sigma
+
+
+def normalization(data):
+    _range = np.max(data) - np.min(data)
+    return (data - np.min(data)) / _range
+
+
+if __name__ == '__main__':
+    getData(10, 10, False)

pytorch_example/RUL/baseModel/dctAttention.py

@@ -105,6 +105,52 @@ class dct_channel_block(nn.Module):
         return x * lr_weight  # result
 
 
+
+class dct_channel_block_withConv(nn.Module):
+    def __init__(self, channel):
+        super(dct_channel_block_withConv, self).__init__()
+        # self.avg_pool = nn.AdaptiveAvgPool1d(1) #innovation
+        self.fc = nn.Sequential(
+            nn.Linear(channel, channel * 2, bias=False),
+            nn.Dropout(p=0.1),
+            nn.ReLU(inplace=True),
+            nn.Linear(channel * 2, channel, bias=False),
+            nn.Sigmoid()
+        )
+
+        # self.dct_norm = nn.LayerNorm([512], eps=1e-6)
+
+        self.dct_norm = nn.LayerNorm([channel], eps=1e-6)  # for lstm on length-wise
+        # self.dct_norm = nn.LayerNorm([36], eps=1e-6)#for lstm on length-wise on ill with input =36
+
+    def forward(self, x):
+        b, c = x.size()  # (B,C,L) (32,96,512)
+
+        # list = []
+        # for i in range(c):
+        #     freq = dct.dct(x[:, :, i])
+        #     list.append(freq)
+        #
+        # stack_dct = torch.stack(list, dim=2)
+
+        # change = x.transpose(2, 1)
+        stack_dct = dct.dct(x,norm='ortho')
+
+        # stack_dct = stack_dct.transpose(2, 1)
+
+        # stack_dct = torch.tensor(stack_dct)
+        '''
+        for traffic mission:f_weight = self.dct_norm(f_weight.permute(0,2,1))#matters for traffic datasets
+        '''
+
+        lr_weight = self.dct_norm(stack_dct)
+        lr_weight = self.fc(stack_dct)
+        lr_weight = self.dct_norm(lr_weight)
+
+        # print("lr_weight",lr_weight.shape)
+        return x * lr_weight  # result
+
+
 if __name__ == '__main__':
     # input_data = torch.Tensor([[1, 2, 3], [4, 5, 6]])  # [2, 3]
     x = torch.rand((32, 10, 64))

pytorch_example/RUL/baseModel/dctChannelAttention.py

@@ -82,26 +82,25 @@ class dct_channel_block(nn.Module):
     def forward(self, x):
         b, t, c = x.size()  # (B,C,L) (32,96,512)
 
-        # list = []
+        list = []
-        # for i in range(c):
+        for i in range(c):
-        #     freq = dct.dct(x[:, :, i])
+            freq = dct.dct(x[:, :, i],norm='ortho')
-        #     list.append(freq)
+            list.append(freq)
-        #
-        # stack_dct = torch.stack(list, dim=2)
 
-        change = x.transpose(2, 1)
+        stack_dct = torch.stack(list, dim=2)
-        stack_dct = dct.dct(change,norm='ortho')
+
-        stack_dct = stack_dct.transpose(2, 1)
+        # 经测试，上下两种不一样
+        # change = x.transpose(2, 1)
+        # stack_dct_a = dct.dct(change,norm='ortho')
+        # stack_dct_b = stack_dct_a.transpose(2, 1)
 
         # stack_dct = torch.tensor(stack_dct)
         '''
         for traffic mission:f_weight = self.dct_norm(f_weight.permute(0,2,1))#matters for traffic datasets
         '''
 
-        lr_weight = self.dct_norm(stack_dct)
         lr_weight = self.fc(stack_dct)
         lr_weight = self.dct_norm(lr_weight)
-
         # print("lr_weight",lr_weight.shape)
         return x * lr_weight  # result
 
@@ -147,12 +146,15 @@ class dct_channel_block1(nn.Module):
 
         # print("lr_weight",lr_weight.shape)
         return x * lr_weight  # result
+
+
 if __name__ == '__main__':
     # input_data = torch.Tensor([[1, 2, 3], [4, 5, 6]])  # [2, 3]
     x = torch.rand((8, 7, 96))
-    dct_model = dct_channel_block1(7)
+    dct_model = dct_channel_block(96)
-    result = dct_model.forward(x)
+    result1,result2 = dct_model.forward(x)
-    print(result)
+    print(result1)
+    print(result2)
     # print(x.shape)
     # m = nn.Linear(64, 2)
     # output = m(x)

pytorch_example/RUL/otherIdea/LSTM/loadData.py

@@ -47,12 +47,12 @@ def normalization(data):
 # LSTM_cell的数目,维度,是否正则化
 def getData(filter_num, dims, if_norm: bool = False):
     # 数据读入
-    HI_merge_data_origin = np.load("../../dataset/HI_merge_data.npy")
+    # HI_merge_data_origin = np.load("../../dataset/HI_merge_data.npy")
-
+    # HI_merge_data = HI_merge_data_origin[0:1250, 1]
     # plt.plot(HI_merge_data[0:1250, 1])
-    # 去除掉退化特征不明显前面的点
+
-    HI_merge_data = HI_merge_data_origin[0:1250, 1]
+
-    # HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\RUL\dataset\smallVHI.csv", delimiter=",")
+    HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\HealthyScorePredict\dataset\Healthy_score.csv", delimiter=",")
 
     # 是否正则化
     if if_norm:

pytorch_example/RUL/otherIdea/LSTM/train.py

@@ -27,12 +27,12 @@ import RUL.baseModel.utils.utils as utils
 超参数设置:
 '''
 hidden_num = 10  # LSTM细胞个数
-feature = 2  # 一个点的维度
+feature = 10  # 一个点的维度
 batch_size = 32
 EPOCH = 1000
 seed = 5
 predict_num = 200  # 预测个数
-is_norm = False
+is_norm = True
 is_single = True
 model_name = "LSTM"
 base_save = r"parameters/{0}_hidden{1}_feature{2}_predict{3}".format(model_name, hidden_num, feature,

pytorch_example/RUL/otherIdea/adaNHDctLSTM/dataset_vibrate/loadData.py

@@ -47,12 +47,11 @@ def normalization(data):
 # LSTM_cell的数目,维度,是否正则化
 def getData(filter_num, dims, if_norm: bool = False):
     # 数据读入
-    HI_merge_data_origin = np.load("E:\self_example\pytorch_example\RUL\dataset\HI_merge_data.npy")
+    # HI_merge_data_origin = np.load("E:\self_example\pytorch_example\RUL\dataset\HI_merge_data.npy")
-
+    # HI_merge_data = HI_merge_data_origin[0:1250, 1]
     # plt.plot(HI_merge_data[0:1250, 1])
-    # 去除掉退化特征不明显前面的点
+
-    HI_merge_data = HI_merge_data_origin[0:1250, 1]
+    HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\HealthyScorePredict\dataset\Healthy_score.csv", delimiter=",")
-    # HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\RUL\dataset\smallVHI.csv", delimiter=",")
     # 是否正则化
     if if_norm:
         HI_merge_data = normalization(HI_merge_data)

pytorch_example/RUL/otherIdea/adaNHDctLSTM/model.py

@@ -60,8 +60,8 @@ class AdaRNN(nn.Module):
                 )
                 ### 不加初始权重会让Hard predict更不稳定，振幅更大
                 # 初始权重越大，振幅越大
-                bottleneck[-1].weight.data.normal_(0, 0.03)
+                # bottleneck[-1].weight.data.normal_(0, 0.03)
-                bottleneck[-1].bias.data.fill_(0.1)
+                # bottleneck[-1].bias.data.fill_(0.1)
                 if bottleneck_list[i][1]:
                     bottleneck.append(nn.BatchNorm1d(bottleneck_list[i][0]))
                 if bottleneck_list[i][2]:
@@ -80,11 +80,11 @@ class AdaRNN(nn.Module):
             nn.Conv1d(in_channels=2, out_channels=64, kernel_size=5, padding='same'),
             nn.BatchNorm1d(64),
             nn.ReLU(),
-            nn.AdaptiveAvgPool1d(2),
+            nn.AdaptiveAvgPool1d(self.len_seq//2),
             nn.Conv1d(in_channels=64, out_channels=64, kernel_size=3, padding='same'),
             nn.BatchNorm1d(64),
             nn.ReLU(),
-            nn.AdaptiveAvgPool1d(2),
+            nn.AdaptiveAvgPool1d(self.len_seq//4),
             nn.Flatten(),
             nn.Dropout(0.2),
             nn.Linear(len_seq // 4 * 64, 64),
@@ -261,9 +261,24 @@ class AdaRNN(nn.Module):
 
 
 if __name__ == '__main__':
-    x = torch.rand((32, 10, 64))
+    x = torch.rand((64, 2, 20))
-    y = x.mean(1)
+
-    z = x.std(1)
+    x=nn.Conv1d(in_channels=2, out_channels=64, kernel_size=5, padding='same')(x)
-    print(y.size())
+    x=nn.BatchNorm1d(64)(x)
-    print(z.size())
+    x=nn.ReLU()(x)
+    x=nn.AdaptiveAvgPool1d(2)(x)
+    x=nn.Conv1d(in_channels=64, out_channels=64, kernel_size=3, padding='same')(x)
+    x=nn.BatchNorm1d(64)(x)
+    x=nn.ReLU()(x)
+    x=nn.AdaptiveAvgPool1d(2)(x)
+    x=nn.Flatten()(x)
+    x=nn.Dropout(0.2)(x)
+    x=nn.Linear(320, 64)(x)
+    x=nn.BatchNorm1d(64)(x)
+    x=nn.ReLU()(x)
+    x=nn.Linear(64, 1)(x)
+    # y = x.mean(1)
+    # z = x.std(1)
+    # print(y.size())
+    # print(z.size())
     pass

pytorch_example/RUL/otherIdea/adaNHDctLSTM/train.py

@@ -30,14 +30,14 @@ from RUL.baseModel.CommonFunction import IsStopTraining
 超参数设置:
 '''
 # 数据准备
-is_norm = False
+is_norm = True
 is_single = True
 tdc_loss_type = 'cos'
 num_domain = 2  # 划分为几个源域和目标域
 
 # RNN相关
 hidden_num = 10  # LSTM细胞个数
-feature = 2  # 一个点的维度
+feature = 20  # 一个点的维度
 predict_num = 200  # 预测个数
 batch_size = 32
 model_name = "AdaRNN"
@@ -55,10 +55,10 @@ epochs = 1000
 transfer_loss_type = 'cos'  # 目前测试cos最好
 dw = 0.5  # 迁移loss权重
 lr = 0.01
-fft_dw = 0.00  # fft_loss权重 整体效果，感觉是让Hard Predict更准了0.01  0.1较好
+fft_dw = 0.01  # fft_loss权重 整体效果，感觉是让Hard Predict更准了0.01  0.1较好
 mdw = 0.0  # 手工特征权重 整体效果，0.3 0.5比较好
 len_win = 0  # 窗口大小为0，暂时不知道有什么用
-seed = 250
+seed = 5
 
 # 相关初始化工作
 out_dir = './outputs'
@@ -362,6 +362,60 @@ def train(model, train_loader_list, valid_loader, lr_patience, early_stop_patien
     pass
 
 
+# 全局最优再保存
+def trainForTotal(model, train_loader_list, valid_loader, lr_patience, early_stop_patience, device):
+    optimizer = optim.SGD(model.parameters(), lr=lr)
+
+    scheduler_model = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="min", factor=0.5,
+                                                                 patience=lr_patience)
+
+    best_score = np.inf
+    best_epoch, stop_round = 0, 0
+    weight_mat, dist_mat = None, None
+    train_loss_list = []
+    val_loss_list = []
+    best_save_path = None
+    for epoch in range(epochs):
+        epoch_start_time = time.time()
+        train_loss, loss1, weight_mat, dist_mat = train_AdaRNN(
+            model, optimizer, train_loader_list, epoch, dist_mat, weight_mat)
+
+        val_loss, val_loss_l1, val_loss_r = val_epoch(
+            model, valid_loader, device=device, scheduler=scheduler_model)
+
+        pprint(
+            "[{:03d}/{:03d}] {:2.2f} sec(s)  train_total_loss: {:3.9f} |  train_mse_loss: {:3.9f} |  train_fft_loss: {:3.9f} |  train_transfer_loss: {:3.9f} "
+            " | val_loss: {:3.9f} | Learning rate : {:3.6f}".format(
+                epoch + 1, epochs, time.time() - epoch_start_time,
+                train_loss[0], train_loss[1], train_loss[3], train_loss[2],
+                val_loss,
+                optimizer.state_dict()['param_groups'][0]['lr']))
+        totalLoss = train_loss[1]+val_loss
+        if len(val_loss_list) == 0 or totalLoss < min(val_loss_list):
+            pprint(f"保存模型最佳模型成功,Loss:{totalLoss}")
+            best_epoch = epoch
+            best_score = val_loss
+            # 保存模型参数
+            if best_save_path != None:
+                utils.delete_file(os.path.join(output_path, best_save_path))
+            best_save_path = save_model_name + "_epoch" + str(epoch) + \
+                             "_trainLoss" + str('%.5f' % train_loss[1]) + \
+                             "_valLoss" + str('%.5f' % val_loss) + ".pkl"
+            print(os.path.join(output_path, best_save_path))
+            torch.save(model.state_dict(),
+                       os.path.join(output_path, best_save_path))
+
+        train_loss_list.append(train_loss)
+        val_loss_list.append(totalLoss)
+
+        if IsStopTraining(history_loss=val_loss_list, patience=early_stop_patience):
+            pprint("{0}次loss未下降,训练停止".format(early_stop_patience))
+            break
+
+    pprint('best val score:', best_score, '@', best_epoch)
+    return best_save_path
+    pass
+
 def main_transfer():
     if torch.cuda.is_available():
         device = torch.device("cuda:0")
@@ -382,6 +436,8 @@ def main_transfer():
     pprint('train model...')
     best_save_path = train(model=model, train_loader_list=train_loader_list, valid_loader=valid_loader, lr_patience=20,
                            early_stop_patience=50, device=device)
+    # best_save_path = trainForTotal(model=model, train_loader_list=train_loader_list, valid_loader=valid_loader, lr_patience=20,
+    #                        early_stop_patience=50, device=device)
 
     end = time.time()
 

pytorch_example/temp/MSETest.py

@@ -0,0 +1,130 @@
+# -*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2023/12/27 16:52
+@Usage : 
+@Desc :  测试MSE三点不足
+'''
+import numpy as np
+import matplotlib.pyplot as plt
+import torch_dct as dct
+
+
+def plot(x, y):
+    font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15}  # 设置坐标标签的字体大小，字体
+    font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15}  # 设置坐标标签的字体大小，字体
+    plt.figure(1)
+    y[-1] = 0.00832
+
+    '''保存的模型参数的路径'''
+
+    from matplotlib import rcParams
+
+    config = {
+        "font.family": 'Times New Roman',  # 设置字体类型
+        "axes.unicode_minus": False,  # 解决负号无法显示的问题
+        "axes.labelsize": 13
+    }
+    rcParams.update(config)
+    predict1 = y[-1] + 0.2
+    predict2 = y[-1] - 0.2
+
+    # 简单预测图
+    plt.scatter(x, y, c='blue', s=16, label='Actual value')
+    plt.plot(x, y, linewidth=2, color='red',
+             label='Traning value')
+    plt.scatter([x[-1]], [predict1], c='black',
+                s=18, label='Predict data1')
+    plt.scatter([x[-1]], [predict2], c='black',
+                s=18, label='Predict data2')
+    plt.xlabel('Serial number of the fusion feature point', font=font1)
+    plt.ylabel('Virtual health indicator', font=font1)
+    plt.legend(loc='upper left', prop=font2)
+
+    plt.show()
+
+    a = np.hstack([y[:-1], predict1])
+
+    b = np.hstack([y[:-1], predict2])
+    return y, a, b
+
+
+def fft(x, a, b):
+    from matplotlib import rcParams
+
+    # 幅值
+    amp_y0 = np.abs(x / len(x))
+    amp_y1 = np.abs(a / len(a))
+    amp_y2 = np.abs(b / len(b))
+    # 相角
+    angle_y0 = np.angle(x)
+    angle_y1 = np.angle(a)
+    angle_y2 = np.angle(b)
+
+    plt.figure(2)
+    config = {
+        "font.family": 'Times New Roman',  # 设置字体类型
+        "axes.unicode_minus": False,  # 解决负号无法显示的问题
+        "axes.labelsize": 13
+    }
+    rcParams.update(config)
+    font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15}  # 设置坐标标签的字体大小，字体
+    font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15}  # 设置坐标标签的字体大小，字体
+
+    plt.plot(amp_y1, linewidth=2, color='green',
+             label='Predict data1')
+    plt.plot(amp_y2, linewidth=2, color='red',
+             label='Predict data2')
+    plt.plot(amp_y0, linewidth=2, color='blue',
+             label='Original data')
+    plt.legend(loc='upper left', prop=font2)
+    plt.xlabel('Serial number of the fusion feature point', font=font1)
+    plt.ylabel('Amplitude', font=font1)
+    plt.show()
+
+    plt.figure(3)
+    config = {
+        "font.family": 'Times New Roman',  # 设置字体类型
+        "axes.unicode_minus": False,  # 解决负号无法显示的问题
+        "axes.labelsize": 13
+    }
+    rcParams.update(config)
+    font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15}  # 设置坐标标签的字体大小，字体
+    font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15}  # 设置坐标标签的字体大小，字体
+
+    plt.plot(angle_y1, linewidth=2, color='green',
+             label='Predict data1')
+    plt.plot(angle_y2, linewidth=2, color='red',
+             label='Predict data2')
+    plt.plot(angle_y0, linewidth=2, color='blue',
+             label='Original data')
+    plt.legend(loc='upper left', prop=font2)
+    plt.xlabel('Serial number of the fusion feature point', font=font1)
+    plt.ylabel('Angle', font=font1)
+    plt.show()
+    pass
+
+
+length = 30
+
+# y = np.array([-0.029078494757,
+#                   -0.33095228672,
+#                   -0.12124221772,
+#                   0.553512275219,
+#                   -0.158036053181,
+#                   0.268739402294,
+#                   -0.638222515583,
+#                   0.233140587807,
+#                   -0.173265621066,
+#                   0.467218101025,
+#                   -0.372010827065,
+#                   -0.136630430818,
+#                   0.343256533146,
+#                   0.008932195604])
+
+y = np.random.random([length, ])
+x = list(range(0, len(y)))
+print(y)
+y, a, b = plot(x, y)
+fft(y, a, b)