，歡迎訪問網(wǎng)頁設(shè)計(jì)愛好者web開發(fā)。c#實(shí)現(xiàn)web信息自動抓取
netbug--爬蟲 v1.02開發(fā)筆記
背景
隨著internet的普及，網(wǎng)絡(luò)信息正以極高的速度增長，在這么多數(shù)據(jù)中找到自己需要的信息是一件很繁瑣的事情，找到需要的信息后如何獲取也是件麻煩的事。這就需要internet信息抓取程序來代替人工的操作。
所謂internet信息抓取程序，就是程序會按照用戶的關(guān)鍵詞或關(guān)鍵網(wǎng)站來收集相應(yīng)的信息，并提供給用戶想要的信息格式。
信息量的增加會帶來信息網(wǎng)站發(fā)布人員工作量的劇增，為實(shí)現(xiàn)信息發(fā)布系統(tǒng)實(shí)現(xiàn)信息自
動發(fā)布、減少工作人員工作量、即時(shí)跟蹤最新信息，就需要自動信息提供程序，因此internet信息抓取程序應(yīng)運(yùn)而生。
目標(biāo)
實(shí)現(xiàn)自定義網(wǎng)站信息分類抓取，存入本地?cái)?shù)據(jù)庫、生成靜態(tài)頁面或其它用戶定義的信息結(jié)構(gòu)，并下載與信息相關(guān)的多媒體文件。
開發(fā)
l 目標(biāo)站點(diǎn)結(jié)構(gòu)分析
本步驟是準(zhǔn)確抓取信息個(gè)關(guān)鍵。
首先要選擇更新頻率高的頁面做為抓取地址，然后分析要抓取內(nèi)容頁面url特點(diǎn)。
然后分析要抓取信息頁面的元素特性，比如標(biāo)題位置，內(nèi)容位置 等，得到定位標(biāo)記點(diǎn)。
將以上信息寫成自己的配置文件或存到數(shù)據(jù)庫中。
每個(gè)網(wǎng)站都需要分析，寫出單獨(dú)的配置文件，供抓取程序使用。
l 信息提取
根據(jù)配置文件取得要抓取頁面url，使用httpwebrequest類獲取內(nèi)容：
//獲取http頁面函數(shù)
public string get_http(string a_strurl,int timeout)
{
string strresult ;
try
{
httpwebrequest myreq = (httpwebrequest)httpwebrequest.create(a_strurl) ;
myreq.timeout = timeout;
httpwebresponse httpwresp = (httpwebresponse)myreq.getresponse();
stream mystream = httpwresp.getresponsestream () ;
streamreader sr = new streamreader(mystream , encoding.default);
stringbuilder strbuilder = new stringbuilder();
while (-1 != sr.peek())
{
strbuilder.append(sr.readline()+"/r/n");
}
strresult = strbuilder.tostring();
}
catch(exception exp)
{
strresult = "錯(cuò)誤：" + exp.message ;
}
return strresult ;
}
獲取頁面內(nèi)容后，分析頁面中連接地址取到要抓取的url：
//處理頁面標(biāo)題和鏈接
public string sniffweburl( string urlstr,string blockb,string blocke )
{
string urlch1 = "";
string urlch2 = "";
int end_n1 = 0;
int end_nums = 0;
int end_nums1 = 0;
int end_nums2 = 0;
int end_nums3 = 0;
string reutstr = "";
string retitle = "";
string ret = "";
try
{
int pos01 = urlstr.indexof( "." );
int pos02 = urlstr.lastindexof( "/" );
if( pos01 < 0 )
{
return "";
}
if( pos02 < 0 )
{
return "";
}
int pos03 = urlstr.indexof( "/",pos01 );
if ( pos03 < 0 )
{
urlch1 = urlstr;
urlch2 = urlstr;
}
else
{
urlch1 = urlstr.substring( 0,pos03 );
urlch2 = urlstr.substring( 0,pos02 );
}
string tmpallstr = new publicfun().get_http( urlstr ,time1);
int pos1 = tmpallstr.indexof( blockb );
int pos2 = tmpallstr.indexof( blocke,pos1 + blockb.length );
if ( pos1>0 && pos2>0 && pos2>pos1 )
{
ret = tmpallstr.substring( pos1 + blockb.length,pos2 - pos1 - blockb.length );
ret = ret.substring( ret.indexof( "<" ));
while( ret.indexof( "<a" ) >= 0 )
{
ret = ret.substring( 0,ret.indexof( "<a" ) ) + "<a" + ret.substring( ret.indexof( "<a" ) + 2 );
}
while( ret.indexof( "</a" ) >=0 )
{
ret = ret.substring( 0,ret.indexof( "</a" ) ) + "</a" + ret.substring( ret.indexof( "</a" ) + 3 );
}
while( ret.indexof( "href=" ) >=0 )
{
ret = ret.substring( 0,ret.indexof( "href=" )) + "href=" + ret.substring( ret.indexof( "href=" ) + 5 );
}
while( ret.indexof( "href=" ) >=0 )
{
ret = ret.substring( 0,ret.indexof( "href=" )) + "href=" + ret.substring( ret.indexof( "href=" ) + 5 );
}
while( ret.indexof( "href='" ) >=0 )
{
ret = ret.substring( 0,ret.indexof( "href='" )) + "href=/"" + ret.substring( ret.indexof( "href='" ) + 6 );
}
}
tmpallstr = ret;
int begin_nums = tmpallstr.indexof( "href=" );
while ( begin_nums >= 0 )
{
string tmpstra = "";
string tmpstrb = tmpallstr.substring( begin_nums + 5,1 );
if ( tmpstrb == "/"" )
{
end_n1 = begin_nums + 6;
if ( ( end_n1 + 1 ) > tmpallstr.length )
{
return "";
}
tmpstra = tmpallstr.substring( begin_nums+6,1 );
}
else
{
end_n1 = begin_nums + 5;
tmpstra = tmpstrb;
}
if ( tmpstra == "#" )
{
tmpallstr = tmpallstr.substring( end_n1 );
begin_nums = tmpallstr.indexof( "href=" );
}
else
{
end_nums1 = tmpallstr.indexof( " ",end_n1 );
end_nums2 = tmpallstr.indexof( ">",end_n1 );
end_nums3 = tmpallstr.indexof( "</a",end_nums2 );
if ( ( end_nums3 >= 0 ) && ( end_nums2 >= 0 ) )
{
retitle = tmpallstr.substring( end_nums2 + 1,end_nums3 - end_nums2 - 1 );
if ( end_nums1 > end_nums2 )
{
end_nums = end_nums2;
}
else
{
if ( end_nums1 < 0 )
{
end_nums = end_nums2;
}
else
{
end_nums = end_nums1;
}
}
string str4 = tmpallstr.substring( end_nums-1, end_nums - end_nums + 1 );
if ( str4 =="/"" || str4 == "'" )
{
end_nums = end_nums - 1;
}
string stotalone = tmpallstr.substring( end_n1,end_nums - end_n1 );
if ( stotalone.indexof( "http://" ) <0 )
{
if ( stotalone.indexof( "/" ) == 0 )
{
stotalone = urlch1 + stotalone;
}
else
{
int linshiintnum = 0;
int flags = 0;
string urlchange = urlstr;;
while( stotalone.indexof( "../" ) >= 0 )
{
stotalone = stotalone.substring( stotalone.indexof( "../" ) + 3 );
linshiintnum = linshiintnum + 1;
flags = flags +1;
}
while( ( urlchange.lastindexof( "/" ) >= 0 ) && ( linshiintnum >= 0 ) )
{
urlchange = urlchange.substring( 0,urlchange.lastindexof( "/" ) );
linshiintnum = linshiintnum - 1;
}
if ( flags == 0 )
{
stotalone = urlch2 + "/" + stotalone;
}
else
{
stotalone = urlchange + "/" + stotalone;
}
}
}
reutstr = reutstr + new publicfun().removehtmlcode( retitle ) + stotalone;
tmpallstr = tmpallstr.substring( end_nums3 + 4 );
begin_nums = tmpallstr.indexof( "href=" );
}
else
{
begin_nums = -1;
}
}
}
return reutstr;
}
catch( exception e)
{
return "";
}
}
得到要抓取內(nèi)容的url后，處理該頁面：
//獲取鏈接內(nèi)容并分類處理
public string getwebcontent( string gatherurl,string suburl,string subtitle,string b_content,string e_content,string b_filter,string e_filter,string root )
{
string tmpallstr = "";
string dfstrb = "";
string dfstre = "";
string repicstr = "";//圖片返回路徑
string recontentstr = "";
string pichtml = "images"; //本地圖片路徑
string urlch1 ="";
string urlch2 ="";
int pos1 = gatherurl.indexof( "." );
int pos2 = gatherurl.lastindexof( "/" );
if( pos1 < 0 )
{
return "";
}
if( pos2 < 0 )
{
return "";
}
int pos3 = gatherurl.indexof( "/",pos1 );
if ( pos3 < 0 )
{
urlch1 = gatherurl;
urlch2 = gatherurl;
}
else
{
urlch1 = gatherurl.substring( 0,pos3 );
urlch2 = gatherurl.substring( 0,pos2 );
}
tmpallstr = new publicfun().get_http( suburl,time1 );
//取稿源
string docfromstr = "";
if ( dfstrb != "" && dfstre != "" )
{
if ( tmpallstr != "" )
{
int b_docf = tmpallstr.indexof( dfstrb );
if ( b_docf > 0 )
{
int e_docf = tmpallstr.indexof( dfstre,b_docf + dfstrb.length );
if ( e_docf > 0 && e_docf > b_docf && e_docf - b_docf < 20 )
{
docfromstr = tmpallstr.substring( b_docf + dfstrb.length, e_docf - b_docf - dfstrb.length );
}
}
}
}
//取內(nèi)容
if ( tmpallstr != "" )
{
int begin_strnum = tmpallstr.indexof( b_content );
if ( begin_strnum < 0 )
{
return "";
}
int end_strnum = tmpallstr.indexof( e_content,begin_strnum + b_content.length );
if ( end_strnum < 0 )
{
return "";
}
string stotalsubm = "";
if ( end_strnum > begin_strnum )
{
stotalsubm = tmpallstr.substring ( begin_strnum,end_strnum - begin_strnum );
}
if ( stotalsubm == "" )
{
return "";
}
//過濾無用信息
int bfnum = stotalsubm.indexof( b_filter );
if ( bfnum > -1 )
{
int efnum = stotalsubm.indexof( e_filter,bfnum );
if ( efnum > -1 )
{
if ( efnum > bfnum )
{
stotalsubm = stotalsubm.substring( 0,bfnum ) + stotalsubm.substring( efnum + e_filter.length );
}
}
}
//格式化圖片標(biāo)記
while( stotalsubm.indexof( "src=" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "src=" ) ) + "src=" + stotalsubm.substring( stotalsubm.indexof( "src=" ) + 4 );
}
while( stotalsubm.indexof( "src=" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "src=" ) ) + "src=" + stotalsubm.substring( stotalsubm.indexof( "src=" ) + 4 );
}
while( stotalsubm.indexof( "src='" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "src='" ) ) + "src=/"" + stotalsubm.substring( stotalsubm.indexof( "src='" ) + 5 );
}
//取圖片地址
int end_n12 = 0;
int end_nums2 = 0;
int begin_nums2 = stotalsubm.indexof( "src=" );
while( begin_nums2 >= 0 )
{
string tmpstr = stotalsubm.substring( begin_nums2 + 4,1 );
if ( tmpstr == "/"" )
{
end_n12 = begin_nums2 + 5;
}
else
{
end_n12 = begin_nums2 + 4;
}
int end_nums2a = stotalsubm.indexof( " ",end_n12 );
int end_nums2b = stotalsubm.indexof( ">",end_n12 );
if ( end_nums2b < 0 )
{
break;
}
if ( end_nums2a > end_nums2b )
{
end_nums2 = end_nums2b;
}
else
{
if (end_nums2a<0)
{
end_nums2 = end_nums2b;
}
else
{
end_nums2 = end_nums2a;
}
}
tmpstr = stotalsubm.substring( end_nums2-1,1 );
if ( tmpstr == "/"" || tmpstr == "'" )
{
end_nums2 = end_nums2 - 1;
}
string tmppicstr = stotalsubm.substring( end_n12,end_nums2 - end_n12 );
if ( tmppicstr.indexof( "http://" ) < 0 )
{
if ( tmppicstr.indexof( "/" ) == 0 )
{
tmppicstr = urlch1 + tmppicstr;
}
else
{
int linshiintnum = 0;
int flags = 0;
string urlchange = suburl;
while( tmppicstr.indexof( "../" ) >= 0 )
{
tmppicstr = tmppicstr.substring( tmppicstr.indexof("../") + 3 );
linshiintnum = linshiintnum + 1;
flags = flags + 1;
}
while( ( urlchange.lastindexof( "/" ) >= 0 ) && ( linshiintnum >= 0 ) )
{
urlchange = urlchange.substring( 0,urlchange.lastindexof( "/" ) );
linshiintnum = linshiintnum - 1;
}
if ( flags == 0 )
{
tmppicstr = urlch2 + "/" + tmppicstr;
}
else
{
tmppicstr = urlchange + "/" + tmppicstr;
}
}
}
//tmppicstr = tmppicstr.tolower();
string tmppicstrtmp = tmppicstr.tolower();
//if ( tmppicstr.indexof( ".jpg" ) > 0 || tmppicstr.indexof( ".gif" ) > 0 || tmppicstr.indexof( ".bmp" ) > 0 )
if ( tmppicstrtmp.indexof( ".jpg" ) > 0 || tmppicstrtmp.indexof( ".gif" ) > 0 || tmppicstrtmp.indexof( ".bmp" ) > 0 )
{
repicstr = repicstr + "||" + tmppicstr ;
int flagn2 = tmppicstr.lastindexof( "/" );
string filen2 = pichtml + tmppicstr.substring( flagn2 );
stotalsubm = stotalsubm.substring( 0,end_nums2 ) + ">******" + filen2 + "******<" + stotalsubm.substring( end_nums2 );
begin_nums2 = stotalsubm.indexof( "src=", end_nums2 + filen2.length + 22 );
}
else
{
begin_nums2 = stotalsubm.indexof( "src=", end_nums2 + 4 );
}
}
if ( repicstr.length > 2 )
repicstr = repicstr.substring(2);
//內(nèi)容處理 格式化關(guān)鍵標(biāo)記
while( stotalsubm.indexof( "<p" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "<p" ) ) + "|****|<" + stotalsubm.substring( stotalsubm.indexof( "<p" ) + 2 );
}
while( stotalsubm.indexof( "<p" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "<p" ) ) + "|****|<" + stotalsubm.substring( stotalsubm.indexof( "<p" ) + 2 );
}
while( stotalsubm.indexof( "</p" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "</p" ) ) + "|****|<" + stotalsubm.substring( stotalsubm.indexof( "</p" ) + 3 );
}
while( stotalsubm.indexof( "</p" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "</p" ) ) + "|****|<" + stotalsubm.substring( stotalsubm.indexof( "</p" ) + 3 );
}
while( stotalsubm.indexof( "<br" ) >=0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "<br" ) ) + "+****+<" + stotalsubm.substring( stotalsubm.indexof( "<br" ) + 3 );
}
while( stotalsubm.indexof( "<br" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "<br" ) ) + "+****+<" + stotalsubm.substring( stotalsubm.indexof( "<br" ) + 3 );
}
while( stotalsubm.indexof( "<br" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "<br" ) ) + "+****+<" + stotalsubm.substring( stotalsubm.indexof( "<br" ) + 3 );
}
while( stotalsubm.indexof( "<br" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "<br" ) ) + "+****+<" + stotalsubm.substring( stotalsubm.indexof( "<br" ) + 3 );
}
//去除html標(biāo)記
int linshiint1 = stotalsubm.indexof( "<" );
int linshiint2 = stotalsubm.indexof( ">" );
if ( linshiint2 < linshiint1 )
{
stotalsubm = stotalsubm.substring( linshiint2 + 1 );
}
int linshiint11 = stotalsubm.lastindexof( "<" );
int linshiint12 = stotalsubm.lastindexof( ">" );
if ( linshiint12 < linshiint11 )
{
stotalsubm = stotalsubm.substring( 0,linshiint12 + 1 );
}
linshiint1 = stotalsubm.indexof( "<" );
while ( linshiint1 >= 0 )
{
linshiint2 = stotalsubm.indexof( ">",linshiint1 );
if ( linshiint2 >= 0 )
{
stotalsubm = stotalsubm.substring( 0,linshiint1 ) + stotalsubm.substring( linshiint2 + 1 );
}
else
{
stotalsubm = stotalsubm.substring( 0,linshiint1 );
}
linshiint1 = stotalsubm.indexof("<");
}
//還原關(guān)鍵標(biāo)記
int linshiint3 = 0;
int linshiint4 = 0;
while( stotalsubm.indexof( "+****+" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "+****+" ) ) + "<br>/n" + stotalsubm.substring( stotalsubm.indexof( "+****+" ) + 9 );
}
while( stotalsubm.indexof( "|****|" ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( "|****|" ) ) + "<br>/n" + stotalsubm.substring( stotalsubm.indexof( "|****|" ) + 9 );
}
while( stotalsubm.indexof( "******" ) >= 0 )
{
linshiint3 = stotalsubm.indexof( "******" ) + 9;
linshiint4 = stotalsubm.indexof( "******",linshiint3 );
if ( linshiint4 >= 0 )
{
int tmppos = stotalsubm.indexof( "******" );
string tmpstr1 = stotalsubm.substring( 0,tmppos );
string tmpstr2 = stotalsubm.substring( linshiint3,linshiint4 - linshiint3 );
string tmpstr3 = stotalsubm.substring( linshiint4 + 9 );
stotalsubm = tmpstr1 + "<img src=" + tmpstr2 + ">" + tmpstr3;
}
else
{
break;
}
}
//去除內(nèi)容中的標(biāo)題
if ( stotalsubm.indexof( subtitle ) >= 0 )
{
stotalsubm = stotalsubm.substring( 0,stotalsubm.indexof( subtitle ) ) + stotalsubm.substring( stotalsubm.indexof( subtitle ) + subtitle.length );
}
recontentstr = stotalsubm;
//調(diào)用下載圖片功能
//下載圖片到指定目錄
string[] img_url = new publicfun().split( repicstr,"||" );
for ( int i=0;i<img_url.length;i++ )
{
if ( img_url[i] != "" )
{
new publicfun().get_img( img_url[i],10000,root + "//images//" + img_url[i].substring( img_url[i].lastindexof("/")+1 ) );
}
}
}
return recontentstr;
}
以上方法返回要取得的信息，包括標(biāo)題內(nèi)容，圖片地址等。
下載頁面中圖片：
//下載圖片
public void get_img(string a_strurl,int timeout,string filepath)
{
try
{
httpwebrequest myreq = (httpwebrequest)httpwebrequest.create(a_strurl) ;
myreq.timeout = timeout;
httpwebresponse httpwresp = (httpwebresponse)myreq.getresponse();
stream mystream = httpwresp.getresponsestream () ;
bitmap map = new bitmap( mystream );
picturebox picb = new picturebox();
picb.image = (image)map;
string path = filepath.substring( 0,filepath.lastindexof( "//" ) );
if (!directory.exists(path))
{
createdir( path );
}
picb.image.save(filepath);
}
catch(exception exp)
{
string ss = exp.message;
writelog( filepath.substring(0,filepath.lastindexof("//")) + "//error.log",a_strurl + "--" + ss + "/r/n");
}
}
l 保存文件或入庫
上面取得的信息可以按自己的要求保存。
****設(shè)計(jì)的時(shí)候沒有使用url按層次循環(huán)抓取，這樣定義抓取url效率更高，速度更快。
測試程序下載：http://bjfile.focus.cn/file/15379/netbugv102.rar
如有建議請發(fā)送email：[email protected] 或msn [email protected]
注：此版本只提供靜態(tài)文件存儲功能，不提供數(shù)據(jù)庫接口，不提供自定義網(wǎng)站功能。
本程序運(yùn)行需要先安裝.net 框架1.1。
