Python基础-网络编程

介绍相关网络编程介绍

urllib模块相关网络操作

解析url

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
"""
urllib.parse 解析url
"""
from urllib.parse import *

# 返回ParseResult
result = urlparse('http://www.crazy.org:80/index.php;yeeku?name=fkit#frag')

print(result)
# ParseResult(scheme='http', netloc='www.crazy.org:80',
# path='/index.php', params='yeeku', query='name=fkit', fragment='frag')

print('scheme:', result.scheme, result[0])
print('主机和端口:', result.netloc, result[1])
print('主机:', result.hostname)
print('端口:', result.port)
print('资源路径:', result.path, result[2])
print('参数:', result.params, result[3])
print('查询字符串：', result.query, result[4])
print('fragment:', result.fragment, result[5])
print(result.geturl())

# urlunparse() 讲ParseResult或者元组恢复成字符串
print('------urlunparse-----')
result = urlunparse(('http', 'www.crazy.org:80',
                     'index.php', 'yeeku', 'name=fkit', 'frag'))
print(result)

# //开头 urlparse可以识别主机 没有scheme
result = urlparse('//www.crazyit.org:80/index.php')
print('scheme:', result.scheme, result[0])
print('主机和端口：', result.netloc, result[1])
print('资源路径', result.path, result[2])
# 没有scheme也没有// 当做资源路径处理
print('------------------------------')
result = urlparse('www.crazyit.org/index.php')
print('scheme:', result.scheme, result[0])
print('主机和端口：', result.netloc, result[1])
print('资源路径', result.path, result[2])

print('---------------------')
# 解析查询字符串 返回dict
result = parse_qs('name=fkit&name=%E6%98%8E%E5%A4%A9&age=12')
print(result)
# 解析查询字符串 返回dict
result = parse_qsl('name=fkit&name=%E6%98%8E%E5%A4%A9&age=12')
print(result)
# 列表形式的请求参数转成字符串
print(urlencode(result))

print('------------urljoin--------------')

# 被拼接url 不是斜杠开头 替换base所包含的path部分
result = urljoin('http://www.crazyit.org/users/login.html', 'help.html')
print(result)
# 被拼接url 根路径/ 开头 拼接到base域名之后
result = urljoin('http://www.crazyit.org/users/login.html', '/help.html')
print(result)
# 被拼接url 绝对路径 // 开头 url拼接到base的scheme之后
result = urljoin('http://www.crazyit.org/users/login.html', '//help.html')
print(result)

发送请求

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
"""
urllib.request 中urlopen()方法
打开指定资源 读取数据
"""
from urllib.request import *
import urllib.parse

# resutl = urlopen("https://www.baidu.com/")
# data = resutl.read(326)
# print(data.decode('utf-8'))

# with urlopen("https://www.baidu.com/") as f:
#     data = f.read(326)
#     print(data.decode('utf-8'))


# 带参数
print("---------------------------------")
with urlopen("http://localhost:9999/python",
             data='测试数据啊'.encode('utf-8')) as f:
    data = f.read().decode('utf-8')
    print(data)

# # get请求
params = urllib.parse.urlencode({'name': '我python', 'password': '98344'})
url = "http://localhost:9999/test?%s" % params
with urlopen(url) as f:
    print(f.read().decode('utf-8'))

# post请求 data
params = urllib.parse.urlencode({'name': '这是post', 'password': 'tew154'})
params = params.encode('utf-8')
with urlopen('http://localhost:9999/test', data=params) as f:
    print(f.read().decode('utf-8'))

# urlopen() 传入urllib.request.Request 对象 实现put、deldete、。。等方法
params = "put请求数据".encode('utf-8')
req = Request(url='http://localhost:9999/put', data=params, method='PUT')
# request可以添加请求头
req.add_header('my_home', 'http://hello.cn.com')
with urlopen(req) as f:
    print(f.status)
    print(f.read().decode('utf-8'))

下载工具类进行下载

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
from urllib.request import *
import threading


class DownUtil:

    def __init__(self, path, target_file, thread_num):
        self.path = path
        self.thread_num = thread_num
        self.target_file = target_file
        self.threads = []

    def download(self):
        req = Request(url=self.path, method='GET')
        req.add_header('Accept', '*/*')
        req.add_header('Charset', 'UTF-8')
        req.add_header('Connection', 'Keep-Alive')
        f = urlopen(req)
        # 要下载文件的大小
        self.file_size = int(dict(f.headers).get('Content-Length', 0))
        f.close()
        # 计算每个线程要下载的资源大小
        current_part_size = self.file_size // self.thread_num + 1
        for i in range(self.thread_num):
            start_pos = i * current_part_size
            t = open(self.target_file, 'wb')
            t.seek(start_pos, 0)
            td = DownThread(self.path, start_pos, current_part_size, t)
            self.threads.append(td)
            td.start()

    def get_complete_rate(self):
        sum_size = 0
        for i in range(self.thread_num):
            sum_size += self.threads[i].length
        return sum_size / self.file_size


class DownThread(threading.Thread):

    def __init__(self, path, start_pos, current_part_size, current_part):
        super().__init__()
        self.path = path
        self.start_pos = start_pos
        # 当前线程负责下载的大小
        self.current_part_size = current_part_size
        # 当前下载的文件块
        self.current_part = current_part
        # 已下载的字节数
        self.length = 0

    def run(self):
        req = Request(url=self.path, method='GET')
        req.add_header('Accept', '*/*')
        req.add_header('Charset', 'UTF-8')
        req.add_header('Connection', 'Keep-Alive')
        f = urlopen(req)
        # 跳过self.start_pos位置
        for i in range(self.start_pos):
            f.read(1)
        while self.length < self.current_part_size:
            data = f.read(1024)
            if data is None or len(data) <= 0:
                break
            self.current_part.write(data)
            self.length += len(data)
        self.current_part.close()
        f.close()

测试下载工具类

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
from DownUtil import *


du = DownUtil(
    'http://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.3.0/kafka-2.3.0-src.tgz',
    'b.tgz', 5)
du.download()


def show_process():
    print('已完成：%.2f' % du.get_complete_rate())
    global t
    if du.get_complete_rate() < 1:
        t = threading.Timer(0.1, show_process)
        t.start()


t = threading.Timer(0.1, show_process)
t.start()

cookie 管理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
"""
cookie 管理 session
1.创建http.cookiejar.CookieJar对象
2.以CookieJar对象为参数
  创建urllib.request.HTTPCookieProcessor对象
3.以HTTPCookieProcessor对象为参数
  调用urllib.request.build_opener()创建OpenerDirector对象
4.使用创建OpenerDirector对象发送请求
"""
from urllib.request import *
import http.cookiejar
import urllib.parse



cookie_jar = http.cookiejar.MozillaCookieJar('a.txt')
cookie_processor = HTTPCookieProcessor(cookie_jar)
opner = build_opener(cookie_processor)

user_agent = r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
headers = {'User-Agent': user_agent, 'Connection': 'keep-alive'}


params = {'loginName': 'waterdev',
          'password': '88f76e7855df2dc1f0753884713e83d7'}
postdata = urllib.parse.urlencode(params).encode()

request = Request('http://******/***/login/toLogin.mvc',
                  data=postdata, headers=headers)
response = opner.open(request)
print(response.read().decode('utf-8'))

# 写入文件
cookie_jar.save(ignore_discard=True, ignore_expires=True)

request = Request(
    'http://******/***/***/todayWarnStatistics.mvc', headers=headers)
response = opner.open(request)
print(response.read().decode())

使用保存的cookie

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
from urllib.request import *
import http.cookiejar
import urllib.parse


cookie_jar = http.cookiejar.MozillaCookieJar('a.txt')
cookie_jar.load('a.txt', ignore_discard=True, ignore_expires=True)
for item in cookie_jar:
    print('name=' + item.name)
    print('value=' + item.value)

cookie_processor = HTTPCookieProcessor(cookie_jar)
opener = build_opener(cookie_processor)

user_agent = r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
headers = {'User-Agent': user_agent, 'Connection': 'keep-alive'}

request = Request(
    'http://*****/****/***/todayWarnStatistics.mvc', headers=headers)
response = opener.open(request)
print(response.read().decode())
TCP socket使用

客户端和服务端创建

1
2
3
4
5
6
7
8
9
"""
socket client 客户端
"""
import socket

s = socket.socket()
s.connect((' ip地址', 3000))
print('-----%s-----' % s.recv(1024).decode('utf-8'))
s.close()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
"""
socket server服务端
"""
import socket

s = socket.socket()
s.bind((' ip地址', 3000))
s.listen()
while True:
    # 返回对于的socket 和地址
    c, addr = s.accept()
    print(c)
    print(addr)
    c.send('你好,你收到了服务器的新年祝福'.encode('utf-8'))
    c.close()

socket加入线程

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
"""
socket 加入线程
"""

import socket
import threading

socket_list = []
ss = socket.socket()
ss.bind((' ip地址', 30000))
ss.listen()


def read_from_client(s):
    try:
        return s.recv(2048).decode('utf-8')
    except Exception:
        socket_list.remove(s)


def server_target(s):
    try:
        while True:
            content = read_from_client(s)
            print(content)
            if content is None:
                break
            for client_s in socket_list:
                client_s.send(content.encode('utf-8'))
    except Exception as e:
        print(e.strerror)


while True:
    s, addr = ss.accept()
    socket_list.append(s)
    threading.Thread(target=server_target, args=(s,)).start()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
"""
socket 加入线程
"""
import socket
import threading

s = socket.socket()
s.connect((' ip地址', 30000))


def read_from_server(s):
    while True:
        print(s.recv(2048).decode('utf-8'))


threading.Thread(target=read_from_server, args=(s,)).start()
while True:
    line = input('->')
    if line is None or line == 'exit':
        break
    s.send(line.encode('utf-8'))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
"""
socket 加入线程
"""
import socket
import threading

s = socket.socket()
s.connect((' ip地址', 30000))


def read_from_server(s):
    while True:
        print(s.recv(2048).decode('utf-8'))


threading.Thread(target=read_from_server, args=(s,)).start()
while True:
    line = input('->')
    if line is None or line == 'exit':
        break
    s.send(line.encode('utf-8'))
UDP 相关操作
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
"""
udp 通信
"""
import socket
PORT = 30000
DATA_LEN = 4096

books = ('疯狂python', '疯狂android', '疯狂kotlin', '疯狂java')

# UDP
s = socket.socket(type=socket.SOCK_DGRAM)
s.bind((' ip地址', PORT))
for i in range(1000):
    data, addr = s.recvfrom(DATA_LEN)
    print(data.decode('utf-8'))
    send_data = books[i % 4].encode('utf-8')
    s.sendto(send_data, addr)
s.close()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
"""
udp 通信
"""
import socket

PORT = 30000
DATA_LEN = 4096
DEST_IP = ' ip地址'

s = socket.socket(type=socket.SOCK_DGRAM)
while True:
    line = input('->')
    if line is None or line == 'exit':
        break
    data = line.encode('utf-8')
    s.sendto(data, (DEST_IP, PORT))
    data = s.recv(DATA_LEN)
    print(data.decode('utf-8'))

s.close()