From d8746d192fc21a251539a559617cb41ba0052a34 Mon Sep 17 00:00:00 2001
From: kevinding1125 <745518019@qq.com>
Date: Wed, 8 Nov 2023 19:14:57 +0800
Subject: [PATCH] =?UTF-8?q?20231108=E7=88=AC=E8=99=AB=E5=AD=A6=E4=B9=A0?=
=?UTF-8?q?=E6=9B=B4=E6=96=B0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../BeautifulSoup库/BeautifulSoupLearning.py | 244 ++++++++
.../BeautifulSoup库/__init__.py | 8 +
.../Pyquery库/__init__.py | 8 +
.../Pyquery库/pyqueryLearning.py | 329 ++++++++++
.../XPath库/XpathLearning.py | 195 ++++++
.../XPath库/__init__.py | 8 +
.../XPath库/test.html | 9 +
Spider/Chapter03_网页数据的提取/__init__.py | 8 +
.../dataETL/NewtonInsert.py | 153 +++++
.../condition_monitoring/dataETL/__init__.py | 0
.../dataETL/lagrangeInsert.py | 96 +++
.../scada_data_process_for_JBYQ_YSD.py | 149 +++++
.../condition_monitoring/data_deal/baseETL.py | 67 ++
.../data_deal/loadData.py | 228 +++++++
.../data_deal/loadData_daban.py | 207 +++++++
.../data_deal/plot_raw_data.py | 60 ++
.../others_idea/CNN_GRU.py | 262 ++++++++
.../others_idea/__init__.py | 0
.../self_try/Joint_Monitoring.py | 526 ++++++++++++++++
.../self_try/Joint_Monitoring_hard.py | 576 ++++++++++++++++++
20 files changed, 3133 insertions(+)
create mode 100644 Spider/Chapter03_网页数据的提取/BeautifulSoup库/BeautifulSoupLearning.py
create mode 100644 Spider/Chapter03_网页数据的提取/BeautifulSoup库/__init__.py
create mode 100644 Spider/Chapter03_网页数据的提取/Pyquery库/__init__.py
create mode 100644 Spider/Chapter03_网页数据的提取/Pyquery库/pyqueryLearning.py
create mode 100644 Spider/Chapter03_网页数据的提取/XPath库/XpathLearning.py
create mode 100644 Spider/Chapter03_网页数据的提取/XPath库/__init__.py
create mode 100644 Spider/Chapter03_网页数据的提取/XPath库/test.html
create mode 100644 Spider/Chapter03_网页数据的提取/__init__.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/dataETL/NewtonInsert.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/dataETL/__init__.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/dataETL/lagrangeInsert.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/dataETL/scada_data_process_for_JBYQ_YSD.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/data_deal/baseETL.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/data_deal/loadData.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/data_deal/loadData_daban.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/data_deal/plot_raw_data.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/others_idea/CNN_GRU.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/others_idea/__init__.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/self_try/Joint_Monitoring.py
create mode 100644 TensorFlow_eaxmple/Model_train_test/condition_monitoring/self_try/Joint_Monitoring_hard.py
diff --git a/Spider/Chapter03_网页数据的提取/BeautifulSoup库/BeautifulSoupLearning.py b/Spider/Chapter03_网页数据的提取/BeautifulSoup库/BeautifulSoupLearning.py
new file mode 100644
index 0000000..6890cb5
--- /dev/null
+++ b/Spider/Chapter03_网页数据的提取/BeautifulSoup库/BeautifulSoupLearning.py
@@ -0,0 +1,244 @@
+# -*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2023/11/8 16:08
+@Usage :
+@Desc :参考 https://github.com/Python3WebSpider/BeautifulSoupTest
+'''
+
+html = """
+The Dormouse's story
+
+The Dormouse's story
+Once upon a time there were three little sisters; and their names were
+,
+Lacie and
+Tillie;
+and they lived at the bottom of a well.
+...
+"""
+from bs4 import BeautifulSoup
+
+
+
+def baseUse():
+ soup = BeautifulSoup(html, 'lxml')
+ print(soup.title) # The Dormouse's story
+ print(type(soup.title)) #
+ print(soup.title.string) # The Dormouse's story
+ print(soup.head) # The Dormouse's story
+ print(soup.p) # The Dormouse's story
+ print(soup.p.name) # 获取节点名称 p
+ print(soup.p.attrs) # 获取属性 {'class': ['title'], 'name': 'dromouse'}
+ print(soup.p.attrs['name']) # 获取属性值 dromouse
+ print(soup.p['name']) # 获取属性值 dromouse
+ print(soup.body.p['name']) # 嵌套选择 dromouse
+
+ print("==========================")
+
+
+def child():
+ html = """
+
+
+ The Dormouse's story
+
+
+
+ Once upon a time there were three little sisters; and their names were
+
+ Elsie
+
+ Lacie
+ and
+ Tillie
+ and they lived at the bottom of a well.
+
+ ...
+ """
+ soup = BeautifulSoup(html, 'lxml')
+ # 子结点
+ for i, child in enumerate(soup.p.children):
+ print(i, child)
+ print("===============================")
+ # 子孙节点
+ for i, child in enumerate(soup.p.descendants):
+ print(i, child)
+ print("===============================")
+
+
+def parent():
+ soup = BeautifulSoup(html, 'lxml')
+ # 父节点
+ print(soup.a.parent)
+ print("===============================")
+ # 祖父节点
+ print(type(soup.a.parents))
+ print(list(enumerate(soup.a.parents)))
+ print("=============================")
+
+
+def brother():
+ html = """
+
+
+
+ Once upon a time there were three little sisters; and their names were
+
+ Elsie
+
+ Hello
+ Lacie
+ and
+ Tillie
+ and they lived at the bottom of a well.
+
+ """
+ # 兄弟节点
+ soup = BeautifulSoup(html, 'lxml')
+ print('Next Sibling', soup.a.next_sibling)
+ print('Prev Sibling', soup.a.previous_sibling)
+ print('Next Siblings', list(enumerate(soup.a.next_siblings)))
+ print('Prev Siblings', list(enumerate(soup.a.previous_siblings)))
+
+# 找到所有满足条件的
+def findAll():
+
+ html = '''
+
+ '''
+ soup = BeautifulSoup(html, 'lxml')
+ print(soup.find_all(name='ul'))
+ print(type(soup.find_all(name='ul')[0]))
+
+ for ul in soup.find_all(name='ul'):
+ print(ul.find_all(name='li'))
+
+ for ul in soup.find_all(name='ul'):
+ print(ul.find_all(name='li'))
+ for li in ul.find_all(name='li'):
+ print(li.string)
+
+
+# 找属性满足匹配得到
+def attrs():
+ html = '''
+
+ '''
+
+ soup = BeautifulSoup(html, 'lxml')
+ print(soup.find_all(attrs={'id': 'list-1'}))
+ print(soup.find_all(attrs={'name': 'elements'}))
+
+ # 常用的属性可以不用attrs传递
+ soup = BeautifulSoup(html, 'lxml')
+ print(soup.find_all(id='list-1'))
+ print(soup.find_all(class_='element'))
+ import re
+ print(soup.find_all(string=re.compile('Foo')))# string等同于text,即里面的具体内容
+
+
+# 返回匹配到的第一个元素
+def find():
+ html = '''
+
+ '''
+ soup = BeautifulSoup(html, 'lxml')
+ print(soup.find(name='ul'))
+ print(type(soup.find(name='ul')))
+ print(soup.find(class_='list'))
+
+# css选择器
+def cssSelect():
+ html = '''
+
+ '''
+
+ soup = BeautifulSoup(html, 'lxml')
+ print(soup.select('.panel .panel-heading'))
+ print(soup.select('ul li'))
+ print(soup.select('#list-2 .element'))
+ print(type(soup.select('ul')[0]))
+
+ # 嵌套选择
+ soup = BeautifulSoup(html, 'lxml')
+ for ul in soup.select('ul'):
+ print(ul.select('li'))
+
+ # 获取属性
+ soup = BeautifulSoup(html, 'lxml')
+ for ul in soup.select('ul'):
+ print(ul['id'])
+ print(ul.attrs['id'])
+
+ # 获取文本
+ soup = BeautifulSoup(html, 'lxml')
+ for li in soup.select('li'):
+ print('Get Text:', li.get_text())
+ print('String:', li.string)
+
+
+
+if __name__ == '__main__':
+ cssSelect()
\ No newline at end of file
diff --git a/Spider/Chapter03_网页数据的提取/BeautifulSoup库/__init__.py b/Spider/Chapter03_网页数据的提取/BeautifulSoup库/__init__.py
new file mode 100644
index 0000000..4cced9b
--- /dev/null
+++ b/Spider/Chapter03_网页数据的提取/BeautifulSoup库/__init__.py
@@ -0,0 +1,8 @@
+#-*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2023/11/8 16:07
+@Usage :
+@Desc :
+'''
\ No newline at end of file
diff --git a/Spider/Chapter03_网页数据的提取/Pyquery库/__init__.py b/Spider/Chapter03_网页数据的提取/Pyquery库/__init__.py
new file mode 100644
index 0000000..2460833
--- /dev/null
+++ b/Spider/Chapter03_网页数据的提取/Pyquery库/__init__.py
@@ -0,0 +1,8 @@
+#-*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2023/11/8 16:54
+@Usage :
+@Desc :
+'''
\ No newline at end of file
diff --git a/Spider/Chapter03_网页数据的提取/Pyquery库/pyqueryLearning.py b/Spider/Chapter03_网页数据的提取/Pyquery库/pyqueryLearning.py
new file mode 100644
index 0000000..af946f6
--- /dev/null
+++ b/Spider/Chapter03_网页数据的提取/Pyquery库/pyqueryLearning.py
@@ -0,0 +1,329 @@
+# -*- encoding:utf-8 -*-
+
+'''
+@Author : dingjiawen
+@Date : 2023/11/8 16:54
+@Usage :
+@Desc :Pyquery学习 参考: https://github.com/Python3WebSpider/PyQueryTest
+'''
+from pyquery import PyQuery as pq
+
+
+# 字符串初始化
+def stringBase():
+ html = '''
+
+ '''
+
+ doc = pq(html)
+ print(doc('li'))
+
+
+# URL初始化
+def URLBase():
+ doc = pq(url='https://cuiqingcai.com')
+ print(doc('title'))
+
+ # 上述代码等同于下面
+ # doc = pq(requests.get('https://cuiqingcai.com').text)
+ # print(doc('title'))
+
+
+# 文件初始化
+def fileBase():
+ doc = pq(filename='demo.html')
+ print(doc('li'))
+
+# 基本的css选择器
+def cssSelect():
+ html = '''
+
+ '''
+ doc = pq(html)
+ print(doc('#container .list li'))
+ print(type(doc('#container .list li')))
+
+ #
+ for item in doc('#container .list li').items():
+ print(item.text())
+
+# 寻找子节点
+def child():
+ html = '''
+
+ '''
+ doc = pq(html)
+ items = doc('.list')
+ print(type(items))
+ print(items)
+ lis = items.find('li')
+ print(type(lis))
+ print(lis)
+ #
+ #
+ lis = items.children()
+ print(type(lis))
+ print(lis)
+
+ #
+ lis = items.children('.active')
+ print(lis)
+
+
+def parent():
+ html = '''
+
+ '''
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ items = doc('.list')
+ container = items.parent()
+ print(type(container))
+ print(container)
+
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ items = doc('.list')
+ parents = items.parents()
+ print(type(parents))
+ print(parents)
+
+ parent = items.parents('.wrap')
+ print(parent)
+
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ li = doc('.list .item-0.active')
+ print(li.siblings())
+
+def brother():
+ html = '''
+
+ '''
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ li = doc('.list .item-0.active')
+ print(li.siblings('.active'))
+
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ li = doc('.item-0.active')
+ print(li)
+ print(str(li))
+
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ # 可能是多个节点
+ lis = doc('li').items()
+ print(type(lis))
+ for li in lis:
+ print(li, type(li))
+
+def attrs():
+ html = '''
+
+ '''
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ a = doc('.item-0.active a')
+ print(a, type(a))
+ print(a.attr('href'))
+
+ a = doc('a')
+ print(a, type(a))
+ print(a.attr('href'))
+ print(a.attr.href)
+
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ a = doc('a')
+ for item in a.items():
+ # 获取属性和文本
+ print(item.attr('href'),item.text())
+
+def getHTML():
+ html = '''
+
+ '''
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ li = doc('li')
+ print(li.html()) # 第一个节点对应的html second item
+ print(li.text()) # 所有匹配的节点的文本 second item third item fourth item fifth item
+ print(type(li.text()))
+
+# 增加或者删除节点的class
+def operateNode():
+ html = '''
+
+ '''
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ li = doc('.item-0.active')
+ print(li)
+ li.removeClass('active')
+ print(li)
+ li.addClass('active')
+ print(li)
+
+ '''
+ third item
+
+ third item
+
+ third item
+ '''
+
+
+
+def operateNodeInformation():
+ html = '''
+
+ '''
+ from pyquery import PyQuery as pq
+ doc = pq(html)
+ li = doc('.item-0.active')
+ print(li)
+ li.attr('name', 'link')
+ print(li)
+ li.text('changed item')
+ print(li)
+ li.html('changed item')
+ print(li)
+ '''
+ third item
+ changed item
+ changed item
+ '''
+
+
+def removeInformation():
+ html = '''
+
+ Hello, World
+
This is a paragraph.
+