Java爬取丁香医生疫情数据并存储至数据库

1、通过页面的url获取html代码

// 根URL

	private static String httpRequset(String requesturl) throws IOException {

		StringBuffer buffer = null;

		BufferedReader bufferedReader = null;

		InputStreamReader inputStreamReader = null;

		InputStream inputStream = null;

		HttpsURLConnection httpsURLConnection = null;

		try {

			URL url = new URL(requesturl);

			httpsURLConnection = (HttpsURLConnection) url.openConnection();

			httpsURLConnection.setDoInput(true);

			httpsURLConnection.setRequestMethod("GET");

			inputStream = httpsURLConnection.getInputStream();

			inputStreamReader = new InputStreamReader(inputStream, "utf-8");

			bufferedReader = new BufferedReader(inputStreamReader);

			buffer = new StringBuffer();

			String str = null;

			while ((str = bufferedReader.readLine()) != null) {

				buffer.append(str);

			}

		} catch (MalformedURLException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

		return buffer.toString();

	}

　　2、获取省市疫情数据

/**

	 * 获取全国各个省市的确诊、死亡和治愈人数

	 *

	 * @return

	 */

	public static String getAreaStat() {

		String url = "https://ncov.dxy.cn/ncovh5/view/pneumonia";

		String htmlResult = "";

		try {

			htmlResult = httpRequset(url);

		} catch (IOException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

		// System.out.println(htmlResult);

		// 正则获取数据

		// 因为html的数据格式看着就像json格式，所以我们正则获取json

		String reg = "window.getAreaStat = (.*?)\\}(?=catch)";

		Pattern totalPattern = Pattern.compile(reg);

		Matcher totalMatcher = totalPattern.matcher(htmlResult);

		String result = "";

		if (totalMatcher.find()) {

			result = totalMatcher.group(1);

			System.out.println(result);

			// 各个省市的是一个列表List，如果想保存到数据库中，要遍历结果，下面是demo

			JSONArray array = JSONArray.parseArray(result);

			try {

				Connection con =BaseConnection.getConnection("VData");

				Statement stmt = con.createStatement();

				Date date=new Date(System.currentTimeMillis());

				for (int i = 0; i <= 30; i++) {

					com.alibaba.fastjson.JSONObject jsonObject = com.alibaba.fastjson.JSONObject

							.parseObject(array.getString(i));

					String provinceName = jsonObject.getString("provinceName");

					String current = jsonObject.getString("currentConfirmedCount");

					String confirmed = jsonObject.getString("confirmedCount");

					String cured = jsonObject.getString("curedCount");

					String dead = jsonObject.getString("deadCount");

					String suspect=jsonObject.getString("suspectedCount");

					stmt.executeUpdate("insert into province values('"+provinceName+"','"+confirmed+"','"+suspect+"','"+cured+

							"','"+dead+"','"+current+"','"+date+"')");

					JSONArray array2 = jsonObject.getJSONArray("cities");

					for (int j = 0; j < array2.size(); j++) {

						com.alibaba.fastjson.JSONObject jsonObject2 = com.alibaba.fastjson.JSONObject

								.parseObject(array2.getString(j));

						String cityname = jsonObject2.getString("cityName");

						String current2 = jsonObject2.getString("currentConfirmedCount");

						String confirmed2 = jsonObject2.getString("confirmedCount");

						String cured2 = jsonObject2.getString("curedCount");

						String dead2 = jsonObject2.getString("deadCount");

						String suspect2 = jsonObject2.getString("suspectedCount");

						System.out.println();

						stmt.executeUpdate("insert into city values('"+cityname+"','"+confirmed2+"','"+suspect2+"','"+cured2+"','"+dead2+"','"+current2+"','"+provinceName+"','"+date+"')");

					}

				}

				stmt.close();

				con.close();

			} catch (SQLException e) {

				// TODO Auto-generated catch block

				e.printStackTrace();

			}

		}

		return result;

	}

　　3、获取世界疫情数据

/**

	 * 世界

	 *

	 * @return

	 */

	public static String getCountryData() {

		String url = "https://ncov.dxy.cn/ncovh5/view/pneumonia";

		String htmlResult = "";

		try {

			htmlResult = httpRequset(url);

		} catch (IOException e) {

			// TODO Auto-generated catch block

			e.printStackTrace();

		}

		String reg = "window.getListByCountryTypeService2true = (.*?)\\}(?=catch)";

		Pattern totalPattern = Pattern.compile(reg);

		Matcher totalMatcher = totalPattern.matcher(htmlResult);

		String result = "";

		if (totalMatcher.find()) {

			result = totalMatcher.group(1);

			System.out.println(result);

			JSONArray array = JSONArray.parseArray(result);

			try {

				Connection con =BaseConnection.getConnection("VData");

				Statement stmt = con.createStatement();

				for(int i=0;i<array.size();i++) {

					com.alibaba.fastjson.JSONObject jsobj=com.alibaba.fastjson.JSONObject.parseObject(array.getString(i));

					if(!jsobj.getString("provinceName").equals("中国")) {

					Date date = new Date(Long.parseLong(jsobj.getString("createTime")));

					String s="insert into contury values('"+jsobj.getString("continents")+"','"+jsobj.getString("provinceName")

					+"','"+jsobj.getString("currentConfirmedCount")+"','"+jsobj.getString("confirmedCount")+"','"+jsobj.getString("suspectedCount")

					+"','"+jsobj.getString("curedCount")+"','"+jsobj.getString("deadCount")+"','"+date+"')";

					stmt.executeUpdate(s);

					}

				}

				stmt.close();

				con.close();

			}catch (Exception e) {

				// TODO: handle exception

			}

		}

		return "";

	}

Java爬取丁香医生疫情数据并存储至数据库的相关教程结束。