lua 以表的方式传入一个list 循环访问表中全部网址 并抓去网页内容 怎么实现
最新回答
假惺惺‘說爱
2024-11-23 18:58:22
将以下代码保存为fetchhtml.lua：
#!/usr/bin/env lua
socket = require "socket"
-- download() 用来抓取网页内容
function download (host, file)
local c = assert(socket.connect(host, 80))
local count = 0
local h = ""
-- counts number of bytes read
c:send("GET " .. file .. " HTTP/1.0\r\n\r\n")
while true do
local s, status = receive(c)
h = h .. s
count = count + #s
if status == "closed" then
html_page[host .. file] = h
break
end
end
c:close()
print(host, file, count)
end
-- receive() 向远端服务器一次请求接收1k字节网页内容
function receive (connection)
connection:settimeout(0.5)
-- do not block
local s, status, partial = connection:receive(2^10)
if status == "timeout" then
coroutine.yield(connection)
end
return s or partial, status
end
-- 采用非抢占多任务机制，避免多个下载任务之间出现阻塞
threads = {}
function get (host, file)
-- create coroutine
local co = coroutine.create(function ()
download(host, file)
end)
-- insert it in the list
table.insert(threads, co)
end
function dispatch ()
local i = 1
while true do
if threads[i] == nil then
if threads[1] == nil then
break
end
i = 1
end
local status, res = coroutine.resume(threads[i])
if not res then
table.remove(threads, i)
else
i = i + 1
end
end
end
-- 程序主体
-- 打开并读取文件，构造list
f = assert(io.open("html_list.txt", "r"))
lines = {}
for line in f:lines() do
lines[#lines + 1] = line
end
f:close()
-- 循环访问表中全部网址，并获取网页内容
html_page = {}
for i, l in ipairs(lines) do
print(i, l)
host = l
file = "/index.html"
get(host,file)
end
dispatch()
-- 获取结果打印出来
for k,v in pairs(html_page) do
print(k)
print(v)
end
编辑文本文件html_list.txt：
www.lua.orgwww.debian.org
测试结果：
moose@debian:~/Code/baidu_knowledge/lua_fetchhtml$ ./fetchhtml.lua
1www.lua.org
2www.debian.orgwww.debian.org /index.html 774www.lua.org /index.html 591www.debian.org/index.html
HTTP/1.1 200 OK
Date: Sun, 29 Mar 2015 09:00:27 GMT
Server: Apache
Last-Modified: Wed, 17 Apr 2013 12:05:34 GMT
ETag: "369d5b7f-1d6-4da8d4d9c01c7"
Accept-Ranges: bytes
Content-Length: 470
Vary: Accept-Encoding
X-Clacks-Overhead: GNU Terry Pratchett
Connection: close
Content-Type: text/html
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">
<HTML>
<HEAD>
<TITLE>Welcome to senfter!</TITLE>
</HEAD>
<BODY>
<H1>Welcome to senfter!</H1>
This is senfter, a system run by and for the <a href="http://www.debian.org/">Debian Project</a>.
She does stuff.
What kind of stuff and who our kind sponsors are you might learn on
<a href="http://db.debian.org/machines.cgi?host=senfter">db.debian.org</a>.
<P>
<HR NOSHADE />
<FONT size="-1">DSA</FONT>
</BODY>
</HTML>www.lua.org/index.html
HTTP/1.0 302 Moved Temporarily
Connection: close
Date: Sun, 29 Mar 2015 09:00:26 GMT
Server: Cherokee
Location:http://www./index.html
Content-Type: text/plain
URI:http://www./index.html
Content-Length: 348
Content-Type: text/html
<!DOCTYPE HTML PUBLIC "-//IETF//DTD HTML 2.0//EN">
<html>
<head><title>302 Moved Temporarily</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
</head>
<body>
<h1>302 Moved Temporarily</h1>
The document has moved <a href="http://www./index.html">here</a>.
<p><hr>
Cherokee web server, Port 80
</body>
</html>
有关知识点参考《Programming in Lua》第三版中以下章节：
-- 5 Functions
-- 9.4 Non-Preemptive Multithreading
-- 22.1 The Simple I/O Model
-- 22.2 The Complete I/O Model
热门标签
