using System;
using System.Data;
using System.Configuration;
using System.Web;
using System.Web.Security;
using System.Web.UI;
using System.Web.UI.WebControls;
using System.Web.UI.WebControls.WebParts;
using System.Web.UI.HtmlControls;
//引入空間
using System.Net;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Collections;
/// <summary>
/// 采集
/// </summary>
public class caiji
{
public caiji()
{
//
// TODO: 在此處添加構(gòu)造函數(shù)邏輯
//
}
/// <summary>
/// 要采集的網(wǎng)頁的連接地址
/// </summary>
/// <param name="url">url</param>
/// <returns></returns>
public static string caijiByUrl(string url,string chargest,string path)
{
string str = GetSourceTextByUrl(url,chargest);
ArrayList lib = new ArrayList();
int i = 0;
//根據(jù)url取得網(wǎng)站域名
Uri uri = new Uri(url);
//Scheme或者協(xié)議，一般為http,Host為取得域名
string baseurl = uri.Scheme + "://" + uri.Host + "/";
//提取出url，包括src等信息
///S匹配任何非空白字符
Regex g = new Regex(@"(src=(""|/')/S+/.(gif|jpg|png|bmp)(""|/'))", RegexOptions.Multiline | RegexOptions.IgnoreCase);
MatchCollection m = g.Matches(str);
foreach (Match math in m)
{
//已經(jīng)提取到圖片的路徑了，但還需要分絕對路徑，相對路徑，以及后綴名是否為圖片，因?yàn)榭赡転?asp,.aspx這些，比如驗(yàn)證碼圖片
string imgUrl = math.Groups[0].Value.ToLower();//轉(zhuǎn)成小寫,=號之間可能有不定的空格
//去除src與單引號，雙引號
imgUrl = imgUrl.Replace("src","");
imgUrl = imgUrl.Replace("/"","");
imgUrl = imgUrl.Replace("'","");
imgUrl = imgUrl.Replace("=","");
imgUrl = imgUrl.Trim();
//路徑處理
if (imgUrl.Substring(0, 4) != "http")
{
//需要判斷是否是絕對路徑還是相對路徑
if (imgUrl.Substring(0, 1) == "/")
{
imgUrl = baseurl + imgUrl;
}
else
{
imgUrl = url.Substring(0,url.LastIndexOf("/") + 1) + imgUrl;
}
}
//判斷元素是否已經(jīng)存在,-1為不存在
if (lib.IndexOf(imgUrl) == -1)
{
lib.Add(imgUrl);
}
}
string str_ = string.Empty;
WebClient client = new WebClient();
for (int j = 0; j < lib.Count; j++)
{
string savepath = path + DateTime.Now.Month + DateTime.Now.Day + DateTime.Now.Minute + DateTime.Now.Second + j + lib[j].ToString().Substring((lib[j].ToString().Length) -4,4);
try
{
client.DownloadFile(new Uri(lib[j].ToString()), savepath);
str_ += lib[j].ToString() + "<br /> 保存路徑為：" + savepath + "<br /><br />";
}
catch (Exception e)
{
str_ += e.Message;
}
}
return str_;
}
public static string GetSourceTextByUrl(string url,string chargest)
{
WebRequest request = WebRequest.Create(url);
request.Timeout = 20000;//20秒超時(shí)
WebResponse response = request.GetResponse();
Stream resStream = response.GetResponseStream();
StreamReader sr = new StreamReader(resStream,Encoding.GetEncoding(chargest));
return sr.ReadToEnd();
}
}
