java小说爬虫

时不遇 · 发表于 2020-8-25 22:23

public class CartoonChapter {
public static CartoonService cartoonService;
public static CartoonCatalogue setDataSource(String cartoonCod,String chapterCod, String dataSource,CartoonService cartoonService) {
CartoonChapter.cartoonService=cartoonService;
SSLHelper.init();
if ("gufengmh8".equals(dataSource)||"36mh".equals(dataSource)) {
return gufengmh8(dataSource,cartoonCod,chapterCod);
}
return null;
}
private static CartoonCatalogue gufengmh8(String dataSource,String cartoonCod,String chapterCod) {
CartoonCatalogue cartoonCatalogue = new CartoonCatalogue();
List<String> list = new ArrayList<>();
int i = 0;
while (true) {
Document document = null;
if (i == 0) {
document = GetDocument.getDocument("https://m."+dataSource+".com/manhua/", cartoonCod, chapterCod);
} else {
document = GetDocument.getDocument("https://m."+dataSource+".com/manhua/", cartoonCod, chapterCod + "-" + i);
}
if (i == 0) {
if("36mh".equals(dataSource)){
Elements elementsByClass = document.getElementsByClass("p10 title3");
cartoonCatalogue.setCatalogueName(elementsByClass.get(0).childNode(1).childNode(1).childNode(1).toString().replace(">",""));
}else{
Elements elementsByClass = document.getElementsByClass("BarTit");
cartoonCatalogue.setCatalogueName(elementsByClass.get(0).text());
}
CartoonCatalogueDto cartoonCatalogue1 = cartoonService.getCartoonCatalogue(cartoonCod,dataSource);
List<CartoonCatalogue> cartoonCatalogues = cartoonCatalogue1.getCartoonCatalogues();
cartoonCatalogues.forEach(e -> {
if (e.getCatalogueCode().toString().equals(chapterCod)) {
cartoonCatalogue.setNextCode(e.getNextCode());
cartoonCatalogue.setUpCode(e.getUpCode());
}
});
}
String src = getSrc(document,dataSource);
if ("https://res.xiaoqinre.com/images/default/cover.png".equals(src)||"https://img001.shmkks.com/images/default/cover.png".equals(src)) {
break;
}
list.add(src);
i++;
}
cartoonCatalogue.setCatalogueSrc(list);
return cartoonCatalogue;
}
private static String getSrc(Document document,String data) {
Elements elementsByClass=null;
Node childNode=null;
if("36mh".equals(data)){
elementsByClass = document.getElementsByClass("UnderPage");
childNode = elementsByClass.get(0).childNode(5).childNode(1);
}else{
elementsByClass = document.getElementsByClass("chapter-content");
childNode = elementsByClass.get(0).childNode(3).childNode(0);
}
String src = null;
List<Node> nodes = childNode.childNodes();
for (Node node : nodes) {
if (!"".equals(node.toString().trim())) {
src = node.attr("src");
}
}
return src;
}
}

复制代码

public class BookChapter {[/font][/color]
public static BookCatalogueDto setDataSource(String dataSource, String bookCod, String chapterCod) {
SSLHelper.init();
if ("biquge5200".equals(dataSource)) {
return biquge5200(bookCod, chapterCod);
} else if ("biquge".equals(dataSource)) {
return biquge(bookCod, chapterCod);
}
return null;
}
private static BookCatalogueDto biquge5200(String bookCod, String chapterCod) {
BookCatalogueDto bookCatalogueDto = new BookCatalogueDto();
Document document = null;
try {
document = Jsoup.connect("https://www.biquge5200.com/" + bookCod + "/" + chapterCod + ".html").get();
} catch (IOException e) {
e.printStackTrace();
}
//<a >下一章</a>
//获得的章节名称
Elements chapterName = document.select("h1");
bookCatalogueDto.setCatalogueName(chapterName.text());
String p = document.select("#content").html();
Elements elementsByClass = document.getElementsByClass("bottem1");
//获取下一章
String node = elementsByClass.get(0).childNode(9).toString();
Pattern pattern = Pattern.compile("<a\\s*href="?([\\w\\W]*?)"?[\\s]*?[^>]>([\\s\\S]*?)(?=</a>)");
Matcher matcher = pattern.matcher(node);
if (matcher.find()) {
String nameCodeUrl = matcher.group(1);
if(!("https://www.biquge5200.com/"+bookCod+"/").equals(nameCodeUrl)) {
String insStr = nameCodeUrl.substring(nameCodeUrl.lastIndexOf("/") + 1, nameCodeUrl.lastIndexOf("."));
bookCatalogueDto.setNextCode(Integer.parseInt(insStr));
}
}
//获取上一章
String node1 = elementsByClass.get(0).childNode(5).toString();
Matcher matcher1 = pattern.matcher(node1);
if (matcher1.find()) {
String nameCodeUrl = matcher1.group(1);
if(!("https://www.biquge5200.com/"+bookCod+"/").equals(nameCodeUrl)){
String insStr = nameCodeUrl.substring(nameCodeUrl.lastIndexOf("/") + 1, nameCodeUrl.lastIndexOf("."));
bookCatalogueDto.setUpCode(Integer.parseInt(insStr));
}
}
String str = p.replace("<div id='content' style='width: 85%;'>", "")
.replace("/n", "")
.replace("</div>", "")
.replace("<p>", "<p data-type="2"><span class="content-wrap">")
.replace("</p>","</span></p>");
bookCatalogueDto.setCatalogueText(str);
bookCatalogueDto.setCatalogueCod(Integer.parseInt(chapterCod));
bookCatalogueDto.setBookCod(bookCod);
return bookCatalogueDto;
}
private static BookCatalogueDto biquge(String bookCod, String chapterCod) {
BookCatalogueDto bookCatalogue = new BookCatalogueDto();
Document document = null;
try {
document = Jsoup.connect("https://www.biquge.com/" + bookCod + "/" + chapterCod + ".html").get();
} catch (Exception e) {
e.printStackTrace();
}
String p = document.select("#content").html();
bookCatalogue.setCatalogueText(p);
bookCatalogue.setCatalogueCod(Integer.parseInt(chapterCod));
Elements chapterName = document.select("h1");
bookCatalogue.setCatalogueName(chapterName.text());
Elements next = document.getElementsByClass("next");
String nextHtml = next.get(0).html();
Pattern pattern = Pattern.compile("<a\\s*href="?([\\w\\W]*?)"?[\\s]*?[^>]>([\\s\\S]*?)(?=</a>)");
Matcher matcher = pattern.matcher(nextHtml);
if (matcher.find()) {
String nameCodeUrl = matcher.group(1);
String insStr = nameCodeUrl.substring(nameCodeUrl.lastIndexOf("/") + 1, nameCodeUrl.lastIndexOf("."));
bookCatalogue.setNextCode(Integer.parseInt(insStr));
}
return bookCatalogue;
}
}

复制代码

yuexiuwen520 · 发表于 2020-8-25 22:23

谢谢大佬

lvyizhen · 发表于 2020-8-25 22:24

好的，非常感谢

释怀12138 · 发表于 2020-8-25 22:38

谢谢分享

tony_lu · 发表于 2020-8-25 23:15

感谢楼主分享

123456qwe · 发表于 2020-8-26 08:14

6666666666

百尺书生 · 发表于 2020-8-26 08:15

厉害了大牛

克索鲁 · 发表于 2020-8-26 09:40

谢谢大牛

eellin · 发表于 2020-8-26 10:22

6666

[java] java小说爬虫

浏览过的版块