源码--用C语言编写一个网络蜘蛛来搜索网上出现的电子邮件地址

分析程序运行中的参数，在链表中添加每个网页地址作为根节点，然后从链表头处理每个节点

对整个链表的处理是先处理兄弟节点，流程图如下：

然后处理每个节点的子节点，流程图如下：

http://zhoulifa.bokee.com/inc/mails3.png

当然，这里采用了递归调用法，在处理子节点数据时，就像处理整个链表一样。

运行过程中应建立树形链表结构，结构图如下：

在启动程序时，分析所带参数，并将每个参数添加到根网页节点中。如果有多个参数，则根网页有兄弟节点。

然后从根节点开始处理这个级别的每个节点，在节点的子节点上添加每个节点网页上的网页链接，处理当前级别的子节点。

源代码如下：

#include

#include

#include

#include

#include

#include

#include

#include

#include

#include

#include

#define USERAGENT "Wget/1.10.2"

#define ACCEPT "*

int port;

char * dir;

char * page;

char * file;

char IsHandled;

struct webnode * brother;

struct webnode * child;

} WEBNODE;

struct sockaddr_in server_addr;

int sockfd = 0, dsend = 0, totalsend = 0, nbytes = 0, reqn = 0, i =

0, j = 0, ret = 0;

struct hostent *host;

char request[409600] = "", buffer[1024] = "", httpheader[1024] =

"";

int FileNumber = 0;

char e[2] = "@/";

WEBNODE * NodeHeader, * NodeTail, * NodeCurr;

char * mapped_mem;

int GetHost(char * , char ** , char ** , int * , char **

);

void AnalyzePage(WEBNODE *);

void AddInitNode(char *, char *, int, char * );

void HandleInitNode(WEBNODE *);

void DisplayNode(WEBNODE *);

void HandOneNode(WEBNODE *);

void DoneWithList(int);

void DoOnce();

void ConnectWeb(void);

void SendRequest(void);

void ReceiveResponse(void);

void GetEmail(char * );

void GetLink(char * );

void GetBeforePos(char * , char ** );

void GetAfterPos(char * , char ** );

void AddChildNode(WEBNODE * , char * );

void GetAfterPosWithSlash(char * , char ** );

void GetMemory(char ** , int );

int IsExistWeb(WEBNODE * , char * , char * , int , char * );

void Rstrchr(char * , int , char ** );

int GetLocalAgent(char * UserAgent, char * Accept, char *

AcceptLanguage, char * AcceptEncoding, char * AcceptCharset, char *

KeepAlive, char * Connection, char * ContentType);

int GetLocalAgent(char * UserAgent, char * Accept, char *

AcceptLanguage, char * AcceptEncoding, char * AcceptCharset, char *

KeepAlive, char * Connection, char * ContentType)

{

memcpy(UserAgent, USERAGENT, strlen(USERAGENT));

memcpy(Accept, ACCEPT, strlen(ACCEPT));

memcpy(AcceptLanguage, ACCEPTLANGUAGE,

strlen(ACCEPTLANGUAGE));

memcpy(AcceptEncoding, ACCEPTENCODING,

strlen(ACCEPTENCODING));

memcpy(AcceptCharset, ACCEPTCHARSET, strlen(ACCEPTCHARSET));

memcpy(KeepAlive, KEEPALIVE, strlen(KEEPALIVE));

memcpy(Connection, CONNECTION, strlen(CONNECTION));

memcpy(ContentType, CONTENTTYPE, strlen(CONTENTTYPE));

return 0;

}

void Rstrchr(char * s, int x, char ** d)

{

int len = strlen(s) - 1;

while(len >= 0) {

if(x == s[len]) {(*d) = s len; return;}

len--;

}

(*d) = 0;

}

void ConnectWeb(void) {

if((sockfd=socket(PF_INET,SOCK_STREAM,0))==-1)

{

fprintf(stderr,"\tSocket Error:%s\a\n",strerror(errno));

exit(1);

}

bzero(&server_addr, sizeof(server_addr));

server_addr.sin_family = AF_INET;

server_addr.sin_port = htons(NodeCurr->port);

server_addr.sin_addr = *((struct in_addr *)host->h_addr);

if(connect(sockfd, (struct sockaddr *)(&server_addr),

sizeof(struct sockaddr)) == -1)

{

fprintf(stderr, "\tConnect Error:%s\a\n", strerror(errno));

exit(1);

}

}

void SendRequest(void) {

dsend = 0;totalsend = 0;

nbytes=strlen(request);

while(totalsend < nbytes) {

dsend = write(sockfd, request totalsend, nbytes -

totalsend);

if(dsend==-1) {fprintf(stderr, "\tsend error!%s\n",

strerror(errno));exit(0);}

totalsend =dsend;

fprintf(stdout, "\ntRequest.%d %d bytes send OK!\n", reqn,

totalsend);

}

}

void ReceiveResponse(void) {

fd_set writefds;

struct timeval tival;

int retry = 0;

FILE * localfp = NULL;

i=0; j = 0;

__ReCeive:

FD_ZERO(&writefds);

tival.tv_sec = 10;

tival.tv_usec = 0;

if(sockfd > 0) FD_SET(sockfd, &writefds);

else {fprintf(stderr, "\n\tError, socket is negative!\n");

exit(0);}

ret = select(sockfd + 1, &writefds, NULL, NULL,

&tival);

if(ret ==0 ) {

if(retry++ < 10) goto __ReCeive;

}

if(ret <= 0) {fprintf(stderr, "\n\tError while receiving!\n");

exit(0);}

if(FD_ISSET(sockfd, &writefds)) {

memset(buffer, 0, 1024);

memset(httpheader, 0, 1024);

if((localfp = fopen(NodeCurr->file, "w")) == NULL) {if(DEBUG)

fprintf(stderr, "create file '%s' error\n", NodeCurr->file);

return;}

while((nbytes=read(sockfd,buffer,1))==1)

{

if(i < 4) {

if(buffer[0] == '\r' || buffer[0] == '\n') i++;

else i = 0;

memcpy(httpheader + j, buffer, 1); j++;

}

else {

fprintf(localfp, "%c", buffer[0]);

//fprintf(stdout, "%c", buffer[0]);

i++;

}

}

fclose(localfp);

}

}

void DoOnce() {

ConnectWeb();

SendRequest();

ReceiveResponse();

close(sockfd);

}

void DoneWithList(int flag) {

if(flag) fprintf(stdout, "\tRequest.%d is:\n%s", ++reqn,

request);

DoOnce();

if(flag) fprintf(stdout, "\n\tThe following is the response

header:\n%s", httpheader);

}

int GetHost(char * src, char ** web, char ** file, int * port, char

** dir) {

char * pA, * pB, * pC;

int len;

*port = 0;

if(!(*src)) return -1;

pA = src;

if(!strncmp(pA, "http://", strlen("http://"))) pA =

src+strlen("http://");

else return 1;

pB = strchr(pA, '/');

if(pB) {

len = strlen(pA) - strlen(pB);

GetMemory(web, len);

memcpy((*web), pA, len);

if(*(pB+1)) {

Rstrchr(pB + 1, '/', &pC);

if(pC) len = strlen(pB + 1) - strlen(pC);

else len = 0;

if(len > 0) {

GetMemory(dir, len);

memcpy((*dir), pB + 1, len);

if(pC + 1) {

len = strlen(pC + 1);

GetMemory(file, len);

memcpy((*file), pC + 1, len);

}

else {

len = 1;

GetMemory(file, len);

memcpy((*file), e, len);

}

}

else {

len = 1;

GetMemory(dir, len);

memcpy((*dir), e + 1, len);

len = strlen(pB + 1);

GetMemory(file, len);

memcpy((*file), pB + 1, len);

}

}

else {

len = 1;

GetMemory(dir, len);

memcpy((*dir), e + 1, len);

len = 1;

GetMemory(file, len);

memcpy((*file), e, len);

}

}

else {

len = strlen(pA);

GetMemory(web, len);

memcpy((*web), pA, strlen(pA));

len = 1;

GetMemory(dir, len);

memcpy((*dir), e + 1, len);

len = 1;

GetMemory(file, len);

memcpy((*file), e, len);

}

pA = strchr((*web), ':');

if(pA) *port = atoi(pA + 1);

else *port = 80;

return 0;

}

int main(int argc, char ** argv)

{

int WebPort;

char * WebHost = 0, * PageAddress = 0, * WebDir = 0;

if(argc < 2) {if(DEBUG) fprintf(stdout, "Command error, you

should input like this:\n\t%s WebPageAddress1 WebPageAddress2

WebPageAddress3 ...", argv[0]); exit(0);}

NodeHeader = NodeTail = NodeCurr = 0;

//setlocale(LC_ALL, "zh_CN.gb2312");

for(i = 1; i < argc; i++) {

ret = GetHost(argv[i], &WebHost, &PageAddress,

&WebPort, &WebDir);

if(ret) {if(DEBUG) fprintf(stdout, "GetHost error from '%s'\n",

argv[i]); exit(0);}

AddInitNode(WebHost, PageAddress, WebPort, WebDir);

}

free(WebHost); free(PageAddress);free(WebDir);

if(DEBUG) {

fprintf(stdout, "\nDisplay.]:", FileNumber);

DisplayNode(NodeHeader);

}

HandleInitNode(NodeHeader);

return 0;

}

void AnalyzePage(WEBNODE * node)

{

int fd;

int flength = 0;

fd = open(node->file, O_RDONLY);

if(fd == -1) goto __AnalyzeDone;

flength = lseek(fd, 1, SEEK_END);

write(fd, "\0", 1);

lseek(fd, 0, SEEK_SET);

mapped_mem = mmap(0, flength, PROT_READ, MAP_PRIVATE, fd, 0);

GetEmail(mapped_mem);

GetLink(mapped_mem);

close(fd);

munmap(mapped_mem, flength);

__AnalyzeDone:

close(fd);

node->IsHandled = 1;

remove(node->file);

}

void AddInitNode(char * Host, char * Page, int Port, char *

Dir)

{

WEBNODE * NewNode;

char filename[MAXFILENAME + 1] = "";

if(NodeHeader == NULL) NewNode = NodeHeader = (WEBNODE

*)malloc(sizeof(WEBNODE));

else NodeTail->brother = NewNode = (WEBNODE

*)malloc(sizeof(WEBNODE));

memset(NewNode, 0, sizeof(WEBNODE));

NewNode->host = (char *)malloc(strlen(Host) + 1);

memset(NewNode->host, 0, strlen(Host) + 1);

NewNode->page = (char *)malloc(strlen(Page) + 1);

memset(NewNode->page, 0, strlen(Page) + 1);

NewNode->dir = (char *)malloc(strlen(Dir) + 1);

memset(NewNode->dir, 0, strlen(Dir) + 1);

NewNode->file = (char *)malloc(MAXFILENAME + 1);

memset(NewNode->file, 0, MAXFILENAME + 1);

strcpy(NewNode->host, Host);

strcpy(NewNode->page, Page);

strcpy(NewNode->dir, Dir);

sprintf(filename, "filed.html", FileNumber++);

strcpy(NewNode->file, filename);

NewNode->port = Port;

NewNode->IsHandled = 0;

NewNode->brother = 0;

NewNode->child = 0;

NodeTail = NewNode;

}

void HandleInitNode(WEBNODE * node)

{

WEBNODE * CurrentNode = 0;

CurrentNode = node;

if(CurrentNode) {

while(CurrentNode) {

if(CurrentNode->IsHandled == 0) {

HandOneNode(CurrentNode);

if(DEBUG) {

fprintf(stdout, "\nDisplay.]:", FileNumber);

DisplayNode(NodeHeader);

}

}

CurrentNode = CurrentNode->brother;

}

CurrentNode = node;

while(CurrentNode) {

if(CurrentNode->child &&

CurrentNode->child->IsHandled == 0) {

HandleInitNode(CurrentNode->child);

}

CurrentNode = CurrentNode->brother;

}

}

}

void DisplayNode(WEBNODE * NodeHeader)

{

WEBNODE * TempNode;

TempNode = NodeHeader;

fprintf(stdout, "\n");

while(TempNode) {

if(!strcmp(TempNode->dir, "/")) fprintf(stdout, "\t%s:%d%s%s

=> %s %d\n", TempNode->host, TempNode->port,

TempNode->dir, strcmp(TempNode->page,

"@")?TempNode->page:"", TempNode->file,

TempNode->IsHandled);

else fprintf(stdout, "\t%s:%d/%s/%s => %s %d\n",

TempNode->host, TempNode->port, TempNode->dir,

strcmp(TempNode->page, "@")?TempNode->page:"",

TempNode->file, TempNode->IsHandled);

TempNode = TempNode->brother;

}

TempNode = NodeHeader;

while(TempNode) {

if(TempNode->child) DisplayNode(TempNode->child);

TempNode = TempNode->brother;

}

}

void HandOneNode(WEBNODE * node)

{

char UserAgent[1024] = "", Accept[1024] = "", AcceptLanguage[1024]

= "", AcceptEncoding[1024] = "", AcceptCharset[1024] = "",

KeepAlive[1024] = "", Connection[1024] = "", ContentType[1024] =

"";

NodeCurr = node;

if((host=gethostbyname(NodeCurr->host))==NULL)

{

if(DEBUG) fprintf(stderr,"\tGethostname '%s' error, %s\n",

NodeCurr->host, strerror(errno));

exit(1);

}

GetLocalAgent(UserAgent, Accept, AcceptLanguage, AcceptEncoding,

AcceptCharset, KeepAlive, Connection, ContentType);

if(strcmp(NodeCurr->dir, "/")) sprintf(request, "GET /%s/%s

HTTP/1.0\r\nHost: %s\r\nUser-Agent: %s\r\nAccept: %s\r\nConnection:

%s\r\n\r\n", NodeCurr->dir, strcmp(NodeCurr->page,

"@")?NodeCurr->page:"", NodeCurr->host, UserAgent, Accept,

Connection);

else sprintf(request, "GET %s%s HTTP/1.0\r\nHost: %s\r\nUser-Agent:

%s\r\nAccept: %s\r\nConnection: %s\r\n\r\n", NodeCurr->dir,

strcmp(NodeCurr->page, "@")?NodeCurr->page:"",

NodeCurr->host, UserAgent, Accept, Connection);

DoneWithList(1);

AnalyzePage(NodeCurr);

}

void GetEmail(char * src)

{

char * pa, * pb, * pc, *pd;

char myemail[1024] = "";

FILE * mailfp = NULL;

if((mailfp = fopen("email.txt", "a+")) == NULL) return;

pa = src;

while((pb = strchr(pa, '@'))) {

GetBeforePos(pb, &pc);

GetAfterPos(pb, &pd);

if(pc && pd && (strlen(pc) > (strlen(pd) + 3)))

{

memset(myemail, 0, 1024);

memcpy(myemail, pc, strlen(pc) - strlen(pd));

if(strcmp(NodeCurr->dir, "/")) fprintf(mailfp,

"%s\thttp://%s/%s/%s\n", myemail, NodeCurr->host,

NodeCurr->dir, strcmp(NodeCurr->page,

"@")?NodeCurr->page:"");

else fprintf(mailfp, "%s\thttp://%s%s%s\n", myemail,

NodeCurr->host, NodeCurr->dir, strcmp(NodeCurr->page,

"@")?NodeCurr->page:"");

if(*(pd + 1)) pa = pd + 1;

else break;

}

else if(*(pb + 1)) pa = pb + 1;

else break;

}

fclose(mailfp);

}

void GetBeforePos(char * src, char ** d)

{

char * x;

if(src - 1) x = src - 1;

else {*d = 0; return ;}

while(x) {

if(*x >= 'a' && *x <= 'z') {x--; continue;}

else if(*x >= 'A' && *x <= 'Z') {x--;

continue;}

else if(*x >= '0' && *x <= '9') {x--;

continue;}

else if(*x == '.' || *x == '-' || *x == '_') {x--; continue;}

else {break;}

}

x++;

if(x) *d = x;

else *d = 0;

}

void GetAfterPos(char * src, char ** d)

{

char * x;

if(src + 1) x = src + 1;

else {*d = 0; return ;}

while(x) {

if(*x >= 'a' && *x <= 'z') {x++; continue;}

else if(*x >= 'A' && *x <= 'Z') {x++;

continue;}

else if(*x >= '0' && *x <= '9') {x++;

continue;}

else if(*x == '.' || *x == '-' || *x == '_') {x++; continue;}

else {break;}

}

if(x) *d = x;

else *d = 0;

}

void GetAfterPosWithSlash(char * src, char ** d)

{

char * x;

if(src) x = src;

else {*d = 0; return ;}

while(x) {

if(*x >= 'a' && *x <= 'z') {x++; continue;}

else if(*x >= 'A' && *x <= 'Z') {x++;

continue;}

else if(*x >= '0' && *x <= '9') {x++;

continue;}

else if(*x == '.' || *x == '-' || *x == '_' || *x == '=') {x++;

continue;}

else if(*x == ':' || *x == '/' || *x == '?' || *x == '&') {x++;

continue;}

else {break;}

}

if(x) *d = x;

else *d = 0;

}

void GetMemory(char ** myanchor, int len)

{

if(!(*myanchor)) (*myanchor) = (char *)malloc(len + 1);

else (*myanchor) = (char *)realloc((void *)(*myanchor), len +

1);

memset((*myanchor), 0, len + 1);

}

void GetLink(char * src)

{

char * pa, * pb, * pc;

char * myanchor = 0;

int len = 0;

pa = src;

do {

if((pb = strstr(pa, "href='"))) {

pc = strchr(pb + 6, '\'');

len = strlen(pb + 6) - strlen(pc);

GetMemory(&myanchor, len);

memcpy(myanchor, pb + 6, len);

}

else if((pb = strstr(pa, "href=\""))) {

pc = strchr(pb + 6, '"');

len = strlen(pb + 6) - strlen(pc);

GetMemory(&myanchor, len);

memcpy(myanchor, pb + 6, len);

}

else if((pb = strstr(pa, "href="))) {

GetAfterPosWithSlash(pb + 5, &pc);

len = strlen(pb + 5) - strlen(pc);

GetMemory(&myanchor, len);

memcpy(myanchor, pb + 5, len);

}

else {goto __returnLink ;}

if(strlen(myanchor) > 0) AddChildNode(NodeCurr, myanchor);

if(pc + 1) pa = pc + 1;

}while(pa);

__returnLink:

return;

}

void AddChildNode(WEBNODE * node, char * src)

{

int WebPort, len;

char * WebHost = 0, * PageAddress = 0, * WebDir = 0, * pC =

0;

WEBNODE * NewNode;

char filename[MAXFILENAME + 1] = "";

char IsFromRoot = 0;

if(!src) return;

if(!strncasecmp(src, "mailto:", strlen("mailto:"))) return ;

if(strstr(src, ".css")) return;

if(strstr(src, ".xml")) return;

if(strstr(src, ".ico")) return;

if(strstr(src, ".jpg")) return;

if(strstr(src, ".gif")) return;

if(strstr(src, "javascript:")) return;

if(strstr(src, "+")) return;

ret = GetHost(src, &WebHost, &PageAddress, &WebPort,

&WebDir);

if(ret) {

len = strlen(node->host);

GetMemory(&WebHost, len);

strcpy(WebHost, node->host);

WebPort = node->port;

IsFromRoot = !strncmp(src, "/", 1);

if(IsFromRoot && (src + 1)) Rstrchr(src + 1, '/',

&pC);

else if(!IsFromRoot) Rstrchr(src, '/', &pC);

else pC = 0;

if(pC) {

if(IsFromRoot) len = strlen(src + 1) - strlen(pC);

else len = strlen(src) - strlen(pC) + strlen(node->dir) +

1;

GetMemory(&WebDir, len);

if(IsFromRoot) memcpy(WebDir, src + 1, len);

else {memcpy(WebDir, node->dir, strlen(node->dir));

strcat(WebDir, "/"); memcpy(WebDir + strlen(node->dir) + 1, src,

strlen(src) - strlen(pC));}

if(pC + 1) {

len = strlen(pC + 1);

GetMemory(&PageAddress, len);

strcpy(PageAddress, pC + 1);

}

else {

len = 1;

GetMemory(&PageAddress, len);

memcpy(PageAddress, e, len);

}

}

else {

if(IsFromRoot) {

len = 1;

GetMemory(&WebDir, len);

memcpy(WebDir, e + 1, len);

len = strlen(src + 1);

GetMemory(&PageAddress, len);

memcpy(PageAddress, src + 1, len);

}

else {

len = strlen(node->dir);

GetMemory(&WebDir, len);

memcpy(WebDir, node->dir, len);

len = strlen(src);

GetMemory(&PageAddress, len);

memcpy(PageAddress, src, len);

}

}

}

ret = IsExistWeb(NodeHeader, WebHost, PageAddress, WebPort,

WebDir);

if(ret) goto __ReturnAdd;

if(node->child == NULL) NewNode = node->child = (WEBNODE

*)malloc(sizeof(WEBNODE));

else NodeTail->brother = NewNode = (WEBNODE

*)malloc(sizeof(WEBNODE));

memset(NewNode, 0, sizeof(WEBNODE));

NewNode->host = (char *)malloc(strlen(WebHost) + 1);

memset(NewNode->host, 0, strlen(WebHost) + 1);

NewNode->page = (char *)malloc(strlen(PageAddress) + 1);

memset(NewNode->page, 0, strlen(PageAddress) + 1);

NewNode->dir = (char *)malloc(strlen(WebDir) + 1);

memset(NewNode->dir, 0, strlen(WebDir) + 1);

NewNode->file = (char *)malloc(MAXFILENAME + 1);

memset(NewNode->file, 0, MAXFILENAME + 1);

strcpy(NewNode->host, WebHost);

strcpy(NewNode->page, PageAddress);

strcpy(NewNode->dir, WebDir);

sprintf(filename, "filed.html", FileNumber++);

strcpy(NewNode->file, filename);

NewNode->port = WebPort;

NewNode->IsHandled = 0;

NewNode->brother = 0;

NewNode->child = 0;

NodeTail = NewNode;

__ReturnAdd:

free(WebHost); free(PageAddress); free(WebDir);

}

int IsExistWeb(WEBNODE * node, char * host, char * page, int port,

char * dir)

{

WEBNODE * t;

t = node;

while(t) {

if(!strcmp(t->host, host) && !strcmp(t->page, page)

&& t->port == port && !strcmp(t->dir, dir))

return 1;

t = t->brother;

}

t = node;

while(t) {

if(t->child) {

ret = IsExistWeb(t->child, host, page, port, dir);

if(ret) return 2;

}

t = t->brother;

}

return 0;

}

编译这个程序：

gcc mailaddrsearch.c -o mailsearcher

输入一个网址作为参数运行一下试试吧：

程序首先找出 http://zhoulifa.bokee.com/5531748.html

页面上的邮件地址保存到当前目录下 email.txt

文件里，每行一条记录，格式为邮件地址和出现该邮件地址的网页。然后分析这个页面上出现的网页链接，把各链接作为子节点加入链表，再去处理子节点，重复上述操作。

这只是一个示例程序，并不完善，如果要使其达到实用的目的，还需要让这个程序效率更高点，比如加入 epoll ( 在 2.4

内核中只有 select 了 ) 实现 I/O 多路复用。又比如对每个子节点实现多线程，每个线程处理一个节点。