seems to made it working
This commit is contained in:
parent
4f8ca9ef16
commit
dd6c6ae03f
1 changed files with 7 additions and 3 deletions
|
@ -78,7 +78,7 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||||
userUrl = getUserUrl(channelDoc);
|
userUrl = getUserUrl(channelDoc);
|
||||||
}
|
}
|
||||||
|
|
||||||
userUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd&live_view=500";
|
userUrl = userUrl + "/videos?veiw=0&flow=list&sort=dd&live_view=10000";
|
||||||
String pageContent = downloader.download(userUrl);
|
String pageContent = downloader.download(userUrl);
|
||||||
doc = Jsoup.parse(pageContent, userUrl);
|
doc = Jsoup.parse(pageContent, userUrl);
|
||||||
nextPageUrl = getNextPageUrl(doc);
|
nextPageUrl = getNextPageUrl(doc);
|
||||||
|
@ -92,8 +92,12 @@ public class YoutubeChannelExtractor extends ChannelExtractor {
|
||||||
doc = Jsoup.parse(htmlDataRaw, nextPageUrl);
|
doc = Jsoup.parse(htmlDataRaw, nextPageUrl);
|
||||||
|
|
||||||
String nextPageHtmlDataRaw = ajaxData.getString("load_more_widget_html");
|
String nextPageHtmlDataRaw = ajaxData.getString("load_more_widget_html");
|
||||||
Document nextPageData = Jsoup.parse(nextPageHtmlDataRaw, nextPageUrl);
|
if(!nextPageHtmlDataRaw.isEmpty()) {
|
||||||
nextPageUrl = getNextPageUrl( nextPageData);
|
Document nextPageData = Jsoup.parse(nextPageHtmlDataRaw, nextPageUrl);
|
||||||
|
nextPageUrl = getNextPageUrl(nextPageData);
|
||||||
|
} else {
|
||||||
|
nextPageUrl = "";
|
||||||
|
}
|
||||||
} catch (JSONException e) {
|
} catch (JSONException e) {
|
||||||
throw new ParsingException("Could not parse json data for next page", e);
|
throw new ParsingException("Could not parse json data for next page", e);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue