网络爬虫模拟登陆获取数据并解析实战（二）

加载中...

网络爬虫模拟登陆获取数据并解析实战（二） | DataLearnerAI

package renren.renren;
public class UserInfo {
	private String authorId;
	private String authorname;
	private String age;
	private String gender;
	private String area;
	private String source;
	private String craw_time;
	public String getAuthorId() {
		return authorId;
	}
	public void setAuthorId(String authorId) {
		this.authorId = authorId;
	}
	public String getAuthorname() {
		return authorname;
	}
	public void setAuthorname(String authorname) {
		this.authorname = authorname;
	}
	public String getAge() {
		return age;
	}
	public void setAge(String age) {
		this.age = age;
	}
	public String getGender() {
		return gender;
	}
	public void setGender(String gender) {
		this.gender = gender;
	}
	public String getArea() {
		 area;
	}
	   {
		.area = area;
	}
	 String  {
		 source;
	}
	   {
		.source = source;
	}
	 String  {
		 craw_time;
	}
	   {
		.craw_time = craw_time;
	}
}

package renren.renren;

import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ResponseHandler;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.BasicResponseHandler;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HTTP;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/*   
 *  合肥工业大学 管理学院 qianyang 1563178220@qq.com
 */
public class Autohome1 {

	private static String AutohomeRenLoginURL = "http://account.autohome.com.cn/Login/ValidIndex";  
	HttpResponse response;  
	private DefaultHttpClient httpclient = new DefaultHttpClient();  
	/*   
	 *  这里是模拟登陆的过程
	 */
	private boolean login() { 
		/*
		 * 用户名及密码
		 * 在我们抓包的过程中，我们发现我们的密码，被MD5加密了，所以这里有一个将密码，转换成MD5的形式
		 */
		String userName  ;
		    ().GetMD5Code();  
		    (AutohomeRenLoginURL);  
		
		List<NameValuePair> nvps =  <NameValuePair>();  
		nvps.add( (, ));  
		nvps.add( (, ));  
		nvps.add( (, ));  
		nvps.add( (, userName));  
		nvps.add( (, password));  
		 {  
			httpost.setEntity( (nvps, HTTP.UTF_8));  
			response = httpclient.execute(httpost);  
		}  (Exception e) {  
			e.printStackTrace();  
			 ;  
		}  {  
			httpost.abort();  
		}  
		 ;  
	}  
	 String  {  
		    (redirectLocation);  
		ResponseHandler<String> responseHandler =  ();  
		   ;  
		 {  
			
			responseBody = httpclient.execute(httpget, responseHandler);
		}  (Exception e) {  
			e.printStackTrace();  
			responseBody = ;  
		}  {  
			httpget.abort();  
		}  
		 responseBody;  
	}  
	 String  {
		String htmlText=;
		 (login()) {  
			 (redirectLocation != ) {
				htmlText=getText(redirectLocation);
			}  
		} 
		 htmlText;
	}  
	
	 List<UserInfo>  { 
		List<UserInfo> UserInfomation =  <UserInfo>();
		    ();  
		String htmltext=autohome.printText(CrawlerUrl);
		   Jsoup.parse(htmltext);
		String authorId=+doc.select().select().attr().replaceAll(,);
		Elements dataElements=doc.select().select();
		String source=;
		Date date= ();
		DateFormat format= ();
		String craw_time=format.format(date);
		String area=;
		String username=;
		String gender=;
		String birthday=;
		 (Element ele:dataElements) {
			 (ele.text().contains()) {
				area=ele.text().replaceAll(Jsoup.parse().text(), ).split()[];
				
			}
			 (ele.text().contains()) {
				gender=ele.text().replaceAll(Jsoup.parse().text(), ).split()[];
				
			}
			 (ele.text().contains()) {
				birthday=ele.text().replaceAll(Jsoup.parse().text(), ).split()[];
				
			}
			 (ele.text().contains()) {
				username=ele.text().replaceAll(Jsoup.parse().text(), ).split()[];
				
			}

		}
		
		    ();
		Info.setAuthorId(authorId);
		Info.setAuthorname(username);
		Info.setAge(birthday);
		Info.setGender(gender);
		Info.setArea(area);
		Info.setSource(source);
		Info.setCraw_time(craw_time);
		UserInfomation.add(Info);
		 UserInfomation;
	}  
	   {
		    (); 
		
		String url=;
		
		List<UserInfo> authorInfo=AutoHome.parser(url);
		
		 (UserInfo userinfo:authorInfo) {
			System.out.println(+userinfo.getAuthorId()++userinfo.getAge()++userinfo.getArea());
		}
	}
}

package renren.renren;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;

public class MD5 {
	 // 全局数组
    private final static String[] strDigits = { "0", "1", "2", "3", "4", "5",
            "6", "7", "8", "9", "a", "b", "c", "d", "e", "f" };
    public MD5() {
    }
    // 返回形式为数字跟字符串
    private static String byteToArrayString(byte bByte) {
        int iRet = bByte;
        // System.out.println("iRet="+iRet);
        if (iRet < 0) {
            iRet += 256;
        }
        int iD1 = iRet / 16;
        int iD2 = iRet % 16;
        return strDigits[iD1] + strDigits[iD2];
    }

    // 返回形式只为数字
    private  String  {
           bByte;
        System.out.println( + iRet);
         (iRet < ) {
            iRet += ;
        }
         String.valueOf(iRet);
    }

    
      String  {
            ();
         (   ; i < bByte.length; i++) {
            sBuffer.append(byteToArrayString(bByte[i]));
        }
         sBuffer.toString();
    }

      String  {
           ;
         {
            resultString =  (strObj);
               MessageDigest.getInstance();
            
            resultString = byteToString(md.digest(strObj.getBytes()));
        }  (NoSuchAlgorithmException ex) {
            ex.printStackTrace();
        }
         resultString;
    }
}

网络爬虫模拟登陆获取数据并解析实战（二）

DataLearner 官方微信

程序的结构

热门博客