论文方面更新

This commit is contained in:
markilue 2024-01-11 15:59:36 +08:00
parent 945a4c120c
commit 4fb91924f6
27 changed files with 684 additions and 129 deletions

View File

@ -185,6 +185,7 @@ public class DataServiceImplTest {
assertEquals(tagName1, result.get(0).getTagName());
assertEquals(tagName2, result.get(1).getTagName());
// if (result.get(0).getTagName().equals(tagName1) && result.get(1).getTagName().equals(tagName2)) {
// DataSample[] datasamples1 = result.get(0).getSamples();
// DataSample[] datasamples2 = result.get(1).getSamples();

View File

@ -15,6 +15,7 @@ class BaseGenerator(object):
self.account_operator = RedisClient(type='account', website=self.website)
self.credential_operator = RedisClient(type='credential', website=self.website)
# 所有子类重写generate方法,写自己的生成逻辑
def generate(self, username, password):
"""
generate method
@ -22,6 +23,7 @@ class BaseGenerator(object):
:param password: password
:return:
"""
# 这里直接抛出了NotImplementedError异常因此子类必须实现该方法否则运行时会报错
raise NotImplementedError
def init(self):
@ -30,6 +32,7 @@ class BaseGenerator(object):
"""
pass
# 模板代码设计模式写标准的run流程
def run(self):
"""
run main process
@ -66,6 +69,7 @@ class Antispider6Generator(BaseGenerator):
return
login_url = 'https://antispider6.scrape.center/login'
s = requests.Session()
# 这里省去了注册账号的过程
s.post(login_url, data={
'username': username,
'password': password
@ -80,7 +84,6 @@ class Antispider6Generator(BaseGenerator):
class Antispider7Generator(BaseGenerator):
MAX_COUNT = 100
def init(self):

View File

@ -24,6 +24,7 @@ def get_conn():
"""
for website in GENERATOR_MAP:
if not hasattr(g, website):
# 这里的get和set本质上可以看成建立一个map
setattr(g, f'{website}_{credential}', RedisClient(credential, website))
setattr(g, f'{website}_{account}', RedisClient(account, website))
return g

View File

@ -10,6 +10,8 @@ class RedisClient(object):
def __init__(self, type, website, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD):
"""
type: 是存储的account信息还是Cookie相关信息
website: 爬取的对应网站
init redis client
:param host: redis host
:param port: redis port
@ -19,8 +21,10 @@ class RedisClient(object):
self.type = type
self.website = website
# 这里的name相当于redis-key
def name(self):
"""
这里是做二级分类二级hash方便存取
get hash name
:return: name of hash
"""

View File

@ -1,6 +1,7 @@
from accountpool.storages.redis import RedisClient
import argparse
# 将生成的账号导入账号池
parser = argparse.ArgumentParser(description='AccountPool')
parser.add_argument('website', type=str, help='website')
args = parser.parse_args()

View File

@ -3,6 +3,7 @@ from acinonyx import run
import requests
from loguru import logger
# 具体用法参考https://pypi.org/project/acinonyx/
# This is a script for registering account for antispider7, using acinonyx to accelerate.
parser = argparse.ArgumentParser(description='AccountPool')
@ -23,6 +24,7 @@ def register(username, password):
if __name__ == '__main__':
accounts = []
# 注册
for index in range(1, 1000):
accounts.append((f'admin{index}', f'admin{index}'))
run(register, accounts)

View File

@ -55,28 +55,28 @@ class model():
# HI指标训练和合成
if __name__ == '__main__':
model = model(input_shape=6).getModel(model_Type='ae')
model.compile(optimizer=tf.optimizers.Adam(0.001), loss=tf.losses.mse, metrics=['acc'])
# model.summary()
checkpoint = tf.keras.callbacks.ModelCheckpoint(
filepath="AE_model.h5",
monitor='val_loss',
verbose=2,
save_best_only=True,
mode='min')
lr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, min_lr=0.0001)
model.compile(optimizer=tf.optimizers.SGD(), loss=tf.losses.mse)
# model.compile(optimizer=tf.optimizers.SGD(learning_rate=0.001), loss=FTMSE())
model.summary()
early_stop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20, mode='min', verbose=1)
history = model.fit(train_data, train_data, epochs=1000, batch_size=100)
HI_merge_data = tf.keras.models.Model(inputs=model.input, outputs=model.get_layer('mid').output).predict(train_data)
print(HI_merge_data)
acc = np.array(history.history.get('acc'))
# if acc[299] > 0.9:
np.save("HI_merge_data1.npy", HI_merge_data)
model.save("AE_model.h5")
# if __name__ == '__main__':
# model = model(input_shape=6).getModel(model_Type='ae')
# model.compile(optimizer=tf.optimizers.Adam(0.001), loss=tf.losses.mse, metrics=['acc'])
# # model.summary()
#
# checkpoint = tf.keras.callbacks.ModelCheckpoint(
# filepath="AE_model.h5",
# monitor='val_loss',
# verbose=2,
# save_best_only=True,
# mode='min')
# lr_scheduler = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=10, min_lr=0.0001)
#
# model.compile(optimizer=tf.optimizers.SGD(), loss=tf.losses.mse)
# # model.compile(optimizer=tf.optimizers.SGD(learning_rate=0.001), loss=FTMSE())
# model.summary()
# early_stop = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=20, mode='min', verbose=1)
#
# history = model.fit(train_data, train_data, epochs=1000, batch_size=100)
# HI_merge_data = tf.keras.models.Model(inputs=model.input, outputs=model.get_layer('mid').output).predict(train_data)
# print(HI_merge_data)
# acc = np.array(history.history.get('acc'))
# # if acc[299] > 0.9:
# np.save("HI_merge_data1.npy", HI_merge_data)
# model.save("AE_model.h5")

View File

@ -48,8 +48,8 @@ if __name__ == '__main__':
0.343256533146,
0.008932195604])
print(tf.signal.fft(array))
print(np.fft.fft(array))
array_fft = np.fft.fft(array)
# print(ifft(array))
print(array_fft)
print(ifft(array))
# print(array_fft)

View File

@ -40,16 +40,16 @@ angle1 = np.arctan2(imag, real)
print(angle)
print(angle1)
# result = []
# (num,) = amp.shape
# (total,) = array.shape
# for j in range(total):
# wk = 2 * np.pi * np.arange(num) / num
# cur = np.sum(amp * np.cos(wk * j + angle))
# result.append(cur)
#
# print(result)
# print(array)
result = []
(num,) = amp.shape
(total,) = array.shape
for j in range(total):
wk = 2 * np.pi * np.arange(num) / num
cur = np.sum(amp * np.cos(wk * j + angle))
result.append(cur)
print(result)
print(array)
# # print(array_fft)
#
# plt.subplot(2, 1, 1)

View File

@ -0,0 +1,33 @@
#-*- encoding:utf-8 -*-
'''
@Author : dingjiawen
@Date : 2024/1/11 14:20
@Usage :
@Desc :
'''
import requests
import numpy as np
import matplotlib.pyplot as plt
def getData():
url = 'http://172.28.9.61:8100/api/data/signalData?dataset_name=Shaft2_radial&thing_id=windturbine-rmq-01&time=20240110200508000'
response = requests.get(url)
y = response.json()['data']['timeSequence_y']
y = np.array(y)
amp = np.abs(np.fft.fft(y) / len(y))
plt.plot(amp[:len(amp)//2])
plt.show()
print(1)
if __name__ == '__main__':
getData()

View File

@ -0,0 +1,85 @@
# -*- encoding:utf-8 -*-
'''
@Author : dingjiawen
@Date : 2023/12/22 19:42
@Usage :
@Desc : 测试计算健康度
'''
import matplotlib.pyplot as plt
import math
import numpy as np
ucl = 4
def func(x, ucl):
y = []
z = []
s = []
g = []
f = []
for i in x:
each_g = -math.exp(math.log(2) / ucl * i) + 2
each_f = 2 / (1 + math.exp((i - ucl) * 1.5)) - 1
if i < ucl:
each_z = (2 / (1 + math.exp(-(ucl / i - 1))) - 1)
else:
each_z = (2 / (1 + math.exp(((i - ucl) / (2 * ucl - i)))) - 1)
each_s = math.cos(math.pi * i / (2 * ucl))
each_y = 50 + 50 * each_z
y.append(each_y)
z.append(each_z)
s.append(each_s)
g.append(each_g)
f.append(each_f)
return y, z, s, g, f
pass
def healthy_score(x):
weight = 2 / (1 + math.exp((x - ucl) * 1.5)) - 1
healthy_score = 60 + 40 * weight
return healthy_score
x = np.linspace(0.1, 2 * ucl - 0.1, 40)
y, z, s, g, f = func(x, 4)
print(x)
print(z)
print(f)
print(s)
print(healthy_score(ucl))
# plt.subplot(4, 1, 1)
# plt.plot(x, y)
# plt.subplot(4, 1, 2)
# plt.plot(x, z)
# plt.subplot(4, 1, 3)
# plt.plot(x, f)
# plt.subplot(4, 1, 4)
# plt.plot(x, s)
# # plt.subplot(4, 1, 4)
# # plt.plot(x, g)
# plt.show()
plt.figure(2)
plt.rc('font', family='Times New Roman') # 全局字体样式
plt.plot(x, f, label='a')
plt.plot(x, z, label='b')
plt.plot(x, s, label='c')
font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 16} # 设置坐标标签的字体大小,字体
font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 13} # 设置坐标标签的字体大小,字体
indices = [i for i in range(8)]
y_indices = [-1, -0.5, 0, 0.5, 1]
x_classes = ['UCL/4', 'UCL/2', '3UCL/4', 'UCL', '5UCL/4', '3UCL/2', '7UCL/4', '2UCL']
y_classes = ['0', '25', '50', '75', '100']
plt.xticks([index + 0.5 for index in indices], x_classes, fontproperties=font1) # 设置横坐标方向rotation=45为45度倾斜
plt.yticks([index for index in y_indices], y_classes, fontproperties=font1) # 设置横坐标方向rotation=45为45度倾斜
# pad调整label与坐标轴之间的距离
plt.tick_params(bottom=True, top=False, left=True, right=False, direction='inout', length=5, width=1, pad=1)
plt.ylabel('Healthy Mapping', fontdict=font2)
plt.xlabel('R$_t$', fontdict=font2)
plt.legend(fontsize=16)
plt.show()

View File

@ -210,28 +210,29 @@ if __name__ == '__main__':
print("===============================")
test_data, test_label = get_training_data(total_data[20000:, :])
predicted_data = newModel.predict(test_data)
rows, cols = predicted_data.shape
print("=====================================")
print(predicted_data)
print(predicted_data.shape)
print("=====================================")
# test_data, test_label = get_training_data(total_data[20000:, :])
# predicted_data = newModel.predict(test_data)
# rows, cols = predicted_data.shape
# print("=====================================")
# print(predicted_data)
# print(predicted_data.shape)
# print("=====================================")
temp = np.abs(predicted_data - test_label)
temp1 = (temp - np.broadcast_to(np.mean(temp, axis=0), shape=predicted_data.shape))
temp2 = np.broadcast_to(np.sqrt(np.var(temp, axis=0)), shape=predicted_data.shape)
temp3 = temp1 / temp2
mse = np.sum((temp1 / temp2) ** 2, axis=1)
print("====================")
print("new_mse:",mse)
print(mse.shape)
np.savetxt("mse", mse, delimiter=',')
print("===================")
# temp = np.abs(predicted_data - test_label)
# temp1 = (temp - np.broadcast_to(np.mean(temp, axis=0), shape=predicted_data.shape))
# temp2 = np.broadcast_to(np.sqrt(np.var(temp, axis=0)), shape=predicted_data.shape)
# temp3 = temp1 / temp2
# mse = np.sum((temp1 / temp2) ** 2, axis=1)
# print("====================")
# print("new_mse:",mse)
# print(mse.shape)
# np.savetxt("mse", mse, delimiter=',')
# print("===================")
plt.plot(mse[2000:])
plt.plot(mean)
plt.plot(max)
# plt.plot(mse[2000:])
plt.plot(mse)
# plt.plot(mean)
# plt.plot(max)
plt.show()
@ -239,24 +240,24 @@ if __name__ == '__main__':
data = pd.DataFrame(mse).ewm(span=3).mean()
print(data)
data =np.array(data)
index,_ = data.shape
for i in range(2396):
if data[i,0] >5:
data[i,0] = data[i-1,:]
print(data)
mean = data[2000:2396,:].mean()
std = data[2000:2396,:].std()
mean=np.broadcast_to(mean,shape=[500,])
std=np.broadcast_to(std,shape=[500,])
plt.plot(data[2000:2396,:])
plt.plot(mean)
plt.plot(mean+3*std)
plt.plot(mean-3*std)
plt.show()
# data = pd.DataFrame(mse).ewm(span=3).mean()
# print(data)
# data =np.array(data)
#
# index,_ = data.shape
#
#
#
# for i in range(2396):
# if data[i,0] >5:
# data[i,0] = data[i-1,:]
# print(data)
# mean = data[2000:2396,:].mean()
# std = data[2000:2396,:].std()
# mean=np.broadcast_to(mean,shape=[500,])
# std=np.broadcast_to(std,shape=[500,])
# plt.plot(data[2000:2396,:])
# plt.plot(mean)
# plt.plot(mean+3*std)
# plt.plot(mean-3*std)
# plt.show()

View File

@ -917,6 +917,7 @@ def plot_loss():
if __name__ == '__main__':
# plot_mse_single()
# model_list = ["DCConv", "GRU"]
#
# for model_name in model_list:
@ -1030,8 +1031,8 @@ if __name__ == '__main__':
# ]
# # acc(list)
#
list=[98.56,98.95,99.95,96.1,95,99.65,76.25,72.64,75.87,68.74,88.36,95.52]
plot_FNR1(list)
# list=[98.56,98.95,99.95,96.1,95,99.65,76.25,72.64,75.87,68.74,88.36,95.52]
# plot_FNR1(list)
# # # #
# # list=[3.43,1.99,1.92,2.17,1.63,1.81,1.78,1.8,0.6]
# list=[3.43,0,0,0,0,0,1.8,0,0,0,0,0,0.6,0]
@ -1048,7 +1049,7 @@ if __name__ == '__main__':
# 单独预测图
# plot_mse('E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\JM_banda/banda_joint_result_predict3.csv',data='E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\JM_banda/raw_data.csv')
# plot_mse_single('E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\RNet_D/banda\RNet_D_banda_mse_predict1.csv')
# plot_mse_single('E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\RNet_3\RNet_3_timestamp120_feature10_result.csv')
plot_mse_single('E:\self_example\TensorFlow_eaxmple\Model_train_test\condition_monitoring\self_try\compare\mse\RNet_3\RNet_3_timestamp120_feature10_result.csv')
# 画3d图
# plot_3d()

View File

@ -25,7 +25,6 @@ z = 0
L.sort(key=lambda x:x.split(".")[1].split("_")[1])
for filename in L:
print("读取了{0}个文件".format(z+1))
# data_single = pd.read_csv(filename, dtype=np.float32, header=None)
data_single = np.loadtxt(filename, delimiter=',',dtype=np.str)
# data_single = data_single.iloc[0, :].values
if z == 0:

View File

@ -0,0 +1,8 @@
#-*- encoding:utf-8 -*-
'''
@Author : dingjiawen
@Date : 2024/1/10 14:55
@Usage :
@Desc :
'''

View File

@ -0,0 +1,8 @@
#-*- encoding:utf-8 -*-
'''
@Author : dingjiawen
@Date : 2024/1/10 14:56
@Usage :
@Desc :
'''

View File

@ -0,0 +1,58 @@
# -*- encoding:utf-8 -*-
'''
@Author : dingjiawen
@Date : 2023/12/22 19:42
@Usage :
@Desc : 测试计算健康度
'''
import matplotlib.pyplot as plt
import math
import numpy as np
def func(x, ucl):
f = []
for i in x:
each_f = 2 / (1 + math.exp((i - ucl) * 1.5)) - 1
f.append(each_f)
return f
pass
def cal_healthy_score(x,ucl):
healthy_score = []
for i in x:
weight = 2 / (1 + math.exp((i - ucl) * 1.5)) - 1
healthy_score.append(50 + 50 * weight)
return healthy_score
if __name__ == '__main__':
ucl = 4
x = np.linspace(0.1, 2 * ucl - 0.1, 40)
y, z, s, g, f = func(x, 4)
plt.figure(2)
plt.rc('font', family='Times New Roman') # 全局字体样式
plt.plot(x, f, label='a')
plt.plot(x, z, label='b')
plt.plot(x, s, label='c')
font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 16} # 设置坐标标签的字体大小,字体
font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 13} # 设置坐标标签的字体大小,字体
indices = [i for i in range(8)]
y_indices = [-1, -0.5, 0, 0.5, 1]
x_classes = ['UCL/4', 'UCL/2', '3UCL/4', 'UCL', '5UCL/4', '3UCL/2', '7UCL/4', '2UCL']
y_classes = ['0', '25', '50', '75', '100']
plt.xticks([index + 0.5 for index in indices], x_classes, fontproperties=font1) # 设置横坐标方向rotation=45为45度倾斜
plt.yticks([index for index in y_indices], y_classes, fontproperties=font1) # 设置横坐标方向rotation=45为45度倾斜
# pad调整label与坐标轴之间的距离
plt.tick_params(bottom=True, top=False, left=True, right=False, direction='inout', length=5, width=1, pad=1)
plt.ylabel('Healthy Mapping', fontdict=font2)
plt.xlabel('R$_t$', fontdict=font2)
plt.legend(fontsize=16)
plt.show()

View File

@ -0,0 +1,102 @@
# -*- encoding:utf-8 -*-
'''
@Author : dingjiawen
@Date : 2024/1/10 14:56
@Usage :
@Desc :
'''
import numpy as np
import matplotlib.pyplot as plt
from healthyScore import cal_healthy_score
def calHealthyScore(data, scope):
mu = np.mean(data[:scope], axis=0)
sigma = np.std(data[:scope], axis=0)
healthy_score = cal_healthy_score(data, mu + 3 * sigma)
print(healthy_score)
# 画健康度图
font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 16} # 设置坐标标签的字体大小,字体
font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 13} # 设置坐标标签的字体大小,字体
plt.rc('font', family='Times New Roman') # 全局字体样式
plt.ylabel('Healthy Score', fontdict=font2)
plt.xlabel('time', fontdict=font2)
# plt.legend(fontsize=16)
plt.plot(healthy_score)
plt.show()
return np.array(healthy_score)
# LSTM_cell的数目,维度,是否正则化
def getData(filter_num, dims, if_norm: bool = False):
# 数据读入
HI_merge_data_origin = np.load("../dataset/HI_merge_data.npy")
HI_merge_data = HI_merge_data_origin[0:1250, 1]
HI_merge_data = calHealthyScore(HI_merge_data, 800)
np.savetxt("../dataset/Healthy_score.csv",HI_merge_data,delimiter=',')
# plt.plot(HI_merge_data)
# plt.show()
# 去除掉退化特征不明显前面的点
# HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\RUL\dataset\smallVHI.csv", delimiter=",")
# # 是否正则化
# if if_norm:
# HI_merge_data = normalization(HI_merge_data)
#
# # plt.plot(HI_merge_data)
# # plt.show()
# (total_dims,) = HI_merge_data.shape
#
# # # 将其分成重叠采样状态-滑动窗口函数
# predict_data = np.empty(shape=[total_dims - filter_num, filter_num])
#
# # 重叠采样获取时间部和训练次数
# for dim in range(total_dims - filter_num):
# predict_data[dim] = HI_merge_data[dim:dim + filter_num]
#
# train_label = predict_data[dims:, :]
# train_label_single = HI_merge_data[dims + filter_num - 1:-1]
#
# # 再重叠采样获取一个点的维度
# '''train_data.shape:(sample,filter_num) -> (sample,filter_num,dims)'''
#
# # # 将其分成重叠采样状态-滑动窗口函数
# train_data = np.empty(shape=[dims, total_dims - filter_num - dims, filter_num])
#
# for dim in range(dims):
# train_data[dim] = predict_data[dim:total_dims - filter_num - dims + dim, :]
#
# # 转置变成想要的数据 (dims,sample,filter_num) -> (sample,filter_num,dims)
#
# train_data = np.transpose(train_data, [1, 2, 0])
#
# # todo 解决模型保存时,query无法序列化的问题
# total_data = HI_merge_data
#
# print("total_data.shape:", total_data.shape)
# print("train_data.shape:", train_data.shape) # (20, 1200, 30)
# print("train_label.shape:", train_label.shape) # (20, 1200)
# print("train_label_single.shape:", train_label_single.shape)
#
# # 所有的原始数据;所有的训练数据;所有的训练标签(预测一个序列);所有的训练标签(预测一个点)
# return total_data, train_data, train_label, train_label_single
def standardization(data):
mu = np.mean(data, axis=0)
sigma = np.std(data, axis=0)
return (data - mu) / sigma
def normalization(data):
_range = np.max(data) - np.min(data)
return (data - np.min(data)) / _range
if __name__ == '__main__':
getData(10, 10, False)

View File

@ -105,6 +105,52 @@ class dct_channel_block(nn.Module):
return x * lr_weight # result
class dct_channel_block_withConv(nn.Module):
def __init__(self, channel):
super(dct_channel_block_withConv, self).__init__()
# self.avg_pool = nn.AdaptiveAvgPool1d(1) #innovation
self.fc = nn.Sequential(
nn.Linear(channel, channel * 2, bias=False),
nn.Dropout(p=0.1),
nn.ReLU(inplace=True),
nn.Linear(channel * 2, channel, bias=False),
nn.Sigmoid()
)
# self.dct_norm = nn.LayerNorm([512], eps=1e-6)
self.dct_norm = nn.LayerNorm([channel], eps=1e-6) # for lstm on length-wise
# self.dct_norm = nn.LayerNorm([36], eps=1e-6)#for lstm on length-wise on ill with input =36
def forward(self, x):
b, c = x.size() # (B,C,L) (32,96,512)
# list = []
# for i in range(c):
# freq = dct.dct(x[:, :, i])
# list.append(freq)
#
# stack_dct = torch.stack(list, dim=2)
# change = x.transpose(2, 1)
stack_dct = dct.dct(x,norm='ortho')
# stack_dct = stack_dct.transpose(2, 1)
# stack_dct = torch.tensor(stack_dct)
'''
for traffic mission:f_weight = self.dct_norm(f_weight.permute(0,2,1))#matters for traffic datasets
'''
lr_weight = self.dct_norm(stack_dct)
lr_weight = self.fc(stack_dct)
lr_weight = self.dct_norm(lr_weight)
# print("lr_weight",lr_weight.shape)
return x * lr_weight # result
if __name__ == '__main__':
# input_data = torch.Tensor([[1, 2, 3], [4, 5, 6]]) # [2, 3]
x = torch.rand((32, 10, 64))

View File

@ -82,26 +82,25 @@ class dct_channel_block(nn.Module):
def forward(self, x):
b, t, c = x.size() # (B,C,L) (32,96,512)
# list = []
# for i in range(c):
# freq = dct.dct(x[:, :, i])
# list.append(freq)
#
# stack_dct = torch.stack(list, dim=2)
list = []
for i in range(c):
freq = dct.dct(x[:, :, i],norm='ortho')
list.append(freq)
change = x.transpose(2, 1)
stack_dct = dct.dct(change,norm='ortho')
stack_dct = stack_dct.transpose(2, 1)
stack_dct = torch.stack(list, dim=2)
# 经测试,上下两种不一样
# change = x.transpose(2, 1)
# stack_dct_a = dct.dct(change,norm='ortho')
# stack_dct_b = stack_dct_a.transpose(2, 1)
# stack_dct = torch.tensor(stack_dct)
'''
for traffic mission:f_weight = self.dct_norm(f_weight.permute(0,2,1))#matters for traffic datasets
'''
lr_weight = self.dct_norm(stack_dct)
lr_weight = self.fc(stack_dct)
lr_weight = self.dct_norm(lr_weight)
# print("lr_weight",lr_weight.shape)
return x * lr_weight # result
@ -147,12 +146,15 @@ class dct_channel_block1(nn.Module):
# print("lr_weight",lr_weight.shape)
return x * lr_weight # result
if __name__ == '__main__':
# input_data = torch.Tensor([[1, 2, 3], [4, 5, 6]]) # [2, 3]
x = torch.rand((8, 7, 96))
dct_model = dct_channel_block1(7)
result = dct_model.forward(x)
print(result)
dct_model = dct_channel_block(96)
result1,result2 = dct_model.forward(x)
print(result1)
print(result2)
# print(x.shape)
# m = nn.Linear(64, 2)
# output = m(x)

View File

@ -47,12 +47,12 @@ def normalization(data):
# LSTM_cell的数目,维度,是否正则化
def getData(filter_num, dims, if_norm: bool = False):
# 数据读入
HI_merge_data_origin = np.load("../../dataset/HI_merge_data.npy")
# HI_merge_data_origin = np.load("../../dataset/HI_merge_data.npy")
# HI_merge_data = HI_merge_data_origin[0:1250, 1]
# plt.plot(HI_merge_data[0:1250, 1])
# 去除掉退化特征不明显前面的点
HI_merge_data = HI_merge_data_origin[0:1250, 1]
# HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\RUL\dataset\smallVHI.csv", delimiter=",")
HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\HealthyScorePredict\dataset\Healthy_score.csv", delimiter=",")
# 是否正则化
if if_norm:

View File

@ -27,12 +27,12 @@ import RUL.baseModel.utils.utils as utils
超参数设置:
'''
hidden_num = 10 # LSTM细胞个数
feature = 2 # 一个点的维度
feature = 10 # 一个点的维度
batch_size = 32
EPOCH = 1000
seed = 5
predict_num = 200 # 预测个数
is_norm = False
is_norm = True
is_single = True
model_name = "LSTM"
base_save = r"parameters/{0}_hidden{1}_feature{2}_predict{3}".format(model_name, hidden_num, feature,

View File

@ -47,12 +47,11 @@ def normalization(data):
# LSTM_cell的数目,维度,是否正则化
def getData(filter_num, dims, if_norm: bool = False):
# 数据读入
HI_merge_data_origin = np.load("E:\self_example\pytorch_example\RUL\dataset\HI_merge_data.npy")
# HI_merge_data_origin = np.load("E:\self_example\pytorch_example\RUL\dataset\HI_merge_data.npy")
# HI_merge_data = HI_merge_data_origin[0:1250, 1]
# plt.plot(HI_merge_data[0:1250, 1])
# 去除掉退化特征不明显前面的点
HI_merge_data = HI_merge_data_origin[0:1250, 1]
# HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\RUL\dataset\smallVHI.csv", delimiter=",")
HI_merge_data = np.loadtxt("E:\self_example\pytorch_example\HealthyScorePredict\dataset\Healthy_score.csv", delimiter=",")
# 是否正则化
if if_norm:
HI_merge_data = normalization(HI_merge_data)

View File

@ -60,8 +60,8 @@ class AdaRNN(nn.Module):
)
### 不加初始权重会让Hard predict更不稳定振幅更大
# 初始权重越大,振幅越大
bottleneck[-1].weight.data.normal_(0, 0.03)
bottleneck[-1].bias.data.fill_(0.1)
# bottleneck[-1].weight.data.normal_(0, 0.03)
# bottleneck[-1].bias.data.fill_(0.1)
if bottleneck_list[i][1]:
bottleneck.append(nn.BatchNorm1d(bottleneck_list[i][0]))
if bottleneck_list[i][2]:
@ -80,11 +80,11 @@ class AdaRNN(nn.Module):
nn.Conv1d(in_channels=2, out_channels=64, kernel_size=5, padding='same'),
nn.BatchNorm1d(64),
nn.ReLU(),
nn.AdaptiveAvgPool1d(2),
nn.AdaptiveAvgPool1d(self.len_seq//2),
nn.Conv1d(in_channels=64, out_channels=64, kernel_size=3, padding='same'),
nn.BatchNorm1d(64),
nn.ReLU(),
nn.AdaptiveAvgPool1d(2),
nn.AdaptiveAvgPool1d(self.len_seq//4),
nn.Flatten(),
nn.Dropout(0.2),
nn.Linear(len_seq // 4 * 64, 64),
@ -261,9 +261,24 @@ class AdaRNN(nn.Module):
if __name__ == '__main__':
x = torch.rand((32, 10, 64))
y = x.mean(1)
z = x.std(1)
print(y.size())
print(z.size())
x = torch.rand((64, 2, 20))
x=nn.Conv1d(in_channels=2, out_channels=64, kernel_size=5, padding='same')(x)
x=nn.BatchNorm1d(64)(x)
x=nn.ReLU()(x)
x=nn.AdaptiveAvgPool1d(2)(x)
x=nn.Conv1d(in_channels=64, out_channels=64, kernel_size=3, padding='same')(x)
x=nn.BatchNorm1d(64)(x)
x=nn.ReLU()(x)
x=nn.AdaptiveAvgPool1d(2)(x)
x=nn.Flatten()(x)
x=nn.Dropout(0.2)(x)
x=nn.Linear(320, 64)(x)
x=nn.BatchNorm1d(64)(x)
x=nn.ReLU()(x)
x=nn.Linear(64, 1)(x)
# y = x.mean(1)
# z = x.std(1)
# print(y.size())
# print(z.size())
pass

View File

@ -30,14 +30,14 @@ from RUL.baseModel.CommonFunction import IsStopTraining
超参数设置:
'''
# 数据准备
is_norm = False
is_norm = True
is_single = True
tdc_loss_type = 'cos'
num_domain = 2 # 划分为几个源域和目标域
# RNN相关
hidden_num = 10 # LSTM细胞个数
feature = 2 # 一个点的维度
feature = 20 # 一个点的维度
predict_num = 200 # 预测个数
batch_size = 32
model_name = "AdaRNN"
@ -55,10 +55,10 @@ epochs = 1000
transfer_loss_type = 'cos' # 目前测试cos最好
dw = 0.5 # 迁移loss权重
lr = 0.01
fft_dw = 0.00 # fft_loss权重 整体效果感觉是让Hard Predict更准了0.01 0.1较好
fft_dw = 0.01 # fft_loss权重 整体效果感觉是让Hard Predict更准了0.01 0.1较好
mdw = 0.0 # 手工特征权重 整体效果0.3 0.5比较好
len_win = 0 # 窗口大小为0暂时不知道有什么用
seed = 250
seed = 5
# 相关初始化工作
out_dir = './outputs'
@ -362,6 +362,60 @@ def train(model, train_loader_list, valid_loader, lr_patience, early_stop_patien
pass
# 全局最优再保存
def trainForTotal(model, train_loader_list, valid_loader, lr_patience, early_stop_patience, device):
optimizer = optim.SGD(model.parameters(), lr=lr)
scheduler_model = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode="min", factor=0.5,
patience=lr_patience)
best_score = np.inf
best_epoch, stop_round = 0, 0
weight_mat, dist_mat = None, None
train_loss_list = []
val_loss_list = []
best_save_path = None
for epoch in range(epochs):
epoch_start_time = time.time()
train_loss, loss1, weight_mat, dist_mat = train_AdaRNN(
model, optimizer, train_loader_list, epoch, dist_mat, weight_mat)
val_loss, val_loss_l1, val_loss_r = val_epoch(
model, valid_loader, device=device, scheduler=scheduler_model)
pprint(
"[{:03d}/{:03d}] {:2.2f} sec(s) train_total_loss: {:3.9f} | train_mse_loss: {:3.9f} | train_fft_loss: {:3.9f} | train_transfer_loss: {:3.9f} "
" | val_loss: {:3.9f} | Learning rate : {:3.6f}".format(
epoch + 1, epochs, time.time() - epoch_start_time,
train_loss[0], train_loss[1], train_loss[3], train_loss[2],
val_loss,
optimizer.state_dict()['param_groups'][0]['lr']))
totalLoss = train_loss[1]+val_loss
if len(val_loss_list) == 0 or totalLoss < min(val_loss_list):
pprint(f"保存模型最佳模型成功,Loss:{totalLoss}")
best_epoch = epoch
best_score = val_loss
# 保存模型参数
if best_save_path != None:
utils.delete_file(os.path.join(output_path, best_save_path))
best_save_path = save_model_name + "_epoch" + str(epoch) + \
"_trainLoss" + str('%.5f' % train_loss[1]) + \
"_valLoss" + str('%.5f' % val_loss) + ".pkl"
print(os.path.join(output_path, best_save_path))
torch.save(model.state_dict(),
os.path.join(output_path, best_save_path))
train_loss_list.append(train_loss)
val_loss_list.append(totalLoss)
if IsStopTraining(history_loss=val_loss_list, patience=early_stop_patience):
pprint("{0}次loss未下降,训练停止".format(early_stop_patience))
break
pprint('best val score:', best_score, '@', best_epoch)
return best_save_path
pass
def main_transfer():
if torch.cuda.is_available():
device = torch.device("cuda:0")
@ -382,6 +436,8 @@ def main_transfer():
pprint('train model...')
best_save_path = train(model=model, train_loader_list=train_loader_list, valid_loader=valid_loader, lr_patience=20,
early_stop_patience=50, device=device)
# best_save_path = trainForTotal(model=model, train_loader_list=train_loader_list, valid_loader=valid_loader, lr_patience=20,
# early_stop_patience=50, device=device)
end = time.time()

View File

@ -0,0 +1,130 @@
# -*- encoding:utf-8 -*-
'''
@Author : dingjiawen
@Date : 2023/12/27 16:52
@Usage :
@Desc : 测试MSE三点不足
'''
import numpy as np
import matplotlib.pyplot as plt
import torch_dct as dct
def plot(x, y):
font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15} # 设置坐标标签的字体大小,字体
font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15} # 设置坐标标签的字体大小,字体
plt.figure(1)
y[-1] = 0.00832
'''保存的模型参数的路径'''
from matplotlib import rcParams
config = {
"font.family": 'Times New Roman', # 设置字体类型
"axes.unicode_minus": False, # 解决负号无法显示的问题
"axes.labelsize": 13
}
rcParams.update(config)
predict1 = y[-1] + 0.2
predict2 = y[-1] - 0.2
# 简单预测图
plt.scatter(x, y, c='blue', s=16, label='Actual value')
plt.plot(x, y, linewidth=2, color='red',
label='Traning value')
plt.scatter([x[-1]], [predict1], c='black',
s=18, label='Predict data1')
plt.scatter([x[-1]], [predict2], c='black',
s=18, label='Predict data2')
plt.xlabel('Serial number of the fusion feature point', font=font1)
plt.ylabel('Virtual health indicator', font=font1)
plt.legend(loc='upper left', prop=font2)
plt.show()
a = np.hstack([y[:-1], predict1])
b = np.hstack([y[:-1], predict2])
return y, a, b
def fft(x, a, b):
from matplotlib import rcParams
# 幅值
amp_y0 = np.abs(x / len(x))
amp_y1 = np.abs(a / len(a))
amp_y2 = np.abs(b / len(b))
# 相角
angle_y0 = np.angle(x)
angle_y1 = np.angle(a)
angle_y2 = np.angle(b)
plt.figure(2)
config = {
"font.family": 'Times New Roman', # 设置字体类型
"axes.unicode_minus": False, # 解决负号无法显示的问题
"axes.labelsize": 13
}
rcParams.update(config)
font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15} # 设置坐标标签的字体大小,字体
font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15} # 设置坐标标签的字体大小,字体
plt.plot(amp_y1, linewidth=2, color='green',
label='Predict data1')
plt.plot(amp_y2, linewidth=2, color='red',
label='Predict data2')
plt.plot(amp_y0, linewidth=2, color='blue',
label='Original data')
plt.legend(loc='upper left', prop=font2)
plt.xlabel('Serial number of the fusion feature point', font=font1)
plt.ylabel('Amplitude', font=font1)
plt.show()
plt.figure(3)
config = {
"font.family": 'Times New Roman', # 设置字体类型
"axes.unicode_minus": False, # 解决负号无法显示的问题
"axes.labelsize": 13
}
rcParams.update(config)
font1 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15} # 设置坐标标签的字体大小,字体
font2 = {'family': 'Times New Roman', 'weight': 'normal', 'size': 15} # 设置坐标标签的字体大小,字体
plt.plot(angle_y1, linewidth=2, color='green',
label='Predict data1')
plt.plot(angle_y2, linewidth=2, color='red',
label='Predict data2')
plt.plot(angle_y0, linewidth=2, color='blue',
label='Original data')
plt.legend(loc='upper left', prop=font2)
plt.xlabel('Serial number of the fusion feature point', font=font1)
plt.ylabel('Angle', font=font1)
plt.show()
pass
length = 30
# y = np.array([-0.029078494757,
# -0.33095228672,
# -0.12124221772,
# 0.553512275219,
# -0.158036053181,
# 0.268739402294,
# -0.638222515583,
# 0.233140587807,
# -0.173265621066,
# 0.467218101025,
# -0.372010827065,
# -0.136630430818,
# 0.343256533146,
# 0.008932195604])
y = np.random.random([length, ])
x = list(range(0, len(y)))
print(y)
y, a, b = plot(x, y)
fft(y, a, b)