Merge remote-tracking branch 'origin/master'

README.md

@@ -36,6 +36,7 @@ NewPipe does not use any Google framework libraries, or the YouTube API. It only
 * Show Next/Related videos
 * Search YouTube in a specific language
 * Orbot/Tor support (no streaming yet, experimental)
+* Watch age restricted material
 
 ### Coming Features
 

app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorRestrictedTest.java

@@ -0,0 +1,84 @@
+package org.schabi.newpipe.extractor.youtube;
+
+import android.test.AndroidTestCase;
+
+import org.schabi.newpipe.Downloader;
+import org.schabi.newpipe.extractor.ExtractionException;
+import org.schabi.newpipe.extractor.ParsingException;
+import org.schabi.newpipe.extractor.VideoInfo;
+import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamExtractor;
+
+import java.io.IOException;
+
+public class YoutubeStreamExtractorRestrictedTest extends AndroidTestCase {
+    private YoutubeStreamExtractor extractor;
+
+    public void setUp() throws IOException, ExtractionException {
+        extractor = new YoutubeStreamExtractor("https://www.youtube.com/watch?v=i6JTvzrpBy0",
+                new Downloader());
+    }
+
+    public void testGetInvalidTimeStamp() throws ParsingException {
+        assertTrue(Integer.toString(extractor.getTimeStamp()),
+                extractor.getTimeStamp() <= 0);
+    }
+
+    public void testGetValidTimeStamp() throws ExtractionException, IOException {
+        YoutubeStreamExtractor extractor =
+                new YoutubeStreamExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader());
+        assertTrue(Integer.toString(extractor.getTimeStamp()),
+                extractor.getTimeStamp() == 174);
+    }
+
+    public void testGetAgeLimit() throws ParsingException {
+        assertTrue(extractor.getAgeLimit() == 18);
+    }
+
+    public void testGetTitle() throws ParsingException {
+        assertTrue(!extractor.getTitle().isEmpty());
+    }
+
+    public void testGetDescription() throws ParsingException {
+        assertTrue(extractor.getDescription() != null);
+    }
+
+    public void testGetUploader() throws ParsingException {
+        assertTrue(!extractor.getUploader().isEmpty());
+    }
+
+    public void testGetLength() throws ParsingException {
+        assertTrue(extractor.getLength() > 0);
+    }
+
+    public void testGetViews() throws ParsingException {
+        assertTrue(extractor.getLength() > 0);
+    }
+
+    public void testGetUploadDate() throws ParsingException {
+        assertTrue(extractor.getUploadDate().length() > 0);
+    }
+
+    public void testGetThumbnailUrl() throws ParsingException {
+        assertTrue(extractor.getThumbnailUrl(),
+                extractor.getThumbnailUrl().contains("https://"));
+    }
+
+    public void testGetUploaderThumbnailUrl() throws ParsingException {
+        assertTrue(extractor.getUploaderThumbnailUrl(),
+                extractor.getUploaderThumbnailUrl().contains("https://"));
+    }
+
+    public void testGetAudioStreams() throws ParsingException {
+        assertTrue(!extractor.getAudioStreams().isEmpty());
+    }
+
+    public void testGetVideoStreams() throws ParsingException {
+        for(VideoInfo.VideoStream s : extractor.getVideoStreams()) {
+            assertTrue(s.url,
+                    s.url.contains("https://"));
+            assertTrue(s.resolution.length() > 0);
+            assertTrue(Integer.toString(s.format),
+                    0 <= s.format && s.format <= 4);
+        }
+    }
+}

app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java

@@ -181,9 +181,13 @@ public class VideoItemDetailFragment extends Fragment {
         }
         @Override
         public void run() {
-            //todo: fix expired thread error:
+            boolean show_age_restricted_content = PreferenceManager.getDefaultSharedPreferences(getActivity())
-            // If the thread calling this runnable is expired, the following function will crash.
+                    .getBoolean(activity.getString(R.string.show_age_restricted_content), false);
-            updateInfo(videoInfo);
+            if(videoInfo.age_limit == 0 || show_age_restricted_content) {
+                updateInfo(videoInfo);
+            } else {
+                onNotSpecifiedContentErrorWithMessage(R.string.video_is_age_restricted);
+            }
         }
     }
 
@@ -418,7 +422,7 @@ public class VideoItemDetailFragment extends Fragment {
             imageLoader.displayImage(info.uploader_thumbnail_url,
                     uploaderThumb, displayImageOptions, new ThumbnailLoadingListener());
         }
-        if(info.thumbnail_url != null && !info.thumbnail_url.isEmpty()) {
+        if(info.thumbnail_url != null && !info.thumbnail_url.isEmpty() && info.next_video != null) {
             imageLoader.displayImage(info.next_video.thumbnail_url,
                     nextVideoThumb, displayImageOptions, new ThumbnailLoadingListener());
         }

app/src/main/java/org/schabi/newpipe/extractor/Parser.java

@@ -1,5 +1,7 @@
 package org.schabi.newpipe.extractor;
 
+import android.util.Log;
+
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
 import java.util.HashMap;
@@ -53,7 +55,11 @@ public class Parser {
         Map<String, String> map = new HashMap<>();
         for(String arg : input.split("&")) {
             String[] split_arg = arg.split("=");
-            map.put(split_arg[0], URLDecoder.decode(split_arg[1], "UTF-8"));
+            if(split_arg.length > 1) {
+                map.put(split_arg[0], URLDecoder.decode(split_arg[1], "UTF-8"));
+            } else {
+                map.put(split_arg[0], "");
+            }
         }
         return map;
     }

app/src/main/java/org/schabi/newpipe/extractor/VideoInfo.java

@@ -52,10 +52,12 @@ public class VideoInfo extends AbstractVideoInfo {
         videoInfo.webpage_url = extractor.getPageUrl();
         videoInfo.id = uiconv.getVideoId(extractor.getPageUrl());
         videoInfo.title = extractor.getTitle();
+        videoInfo.age_limit = extractor.getAgeLimit();
 
         if((videoInfo.webpage_url == null || videoInfo.webpage_url.isEmpty())
                 || (videoInfo.id == null || videoInfo.id.isEmpty())
-                || (videoInfo.title == null /* videoInfo.title can be empty of course */));
+                || (videoInfo.title == null /* videoInfo.title can be empty of course */)
+                || (videoInfo.age_limit == -1));
 
         return videoInfo;
     }
@@ -192,6 +194,11 @@ public class VideoInfo extends AbstractVideoInfo {
         } catch(Exception e) {
             videoInfo.addException(e);
         }
+        try {
+
+        } catch (Exception e) {
+            videoInfo.addException(e);
+        }
 
         return videoInfo;
     }
@@ -209,7 +216,7 @@ public class VideoInfo extends AbstractVideoInfo {
     public String dashMpdUrl = "";
     public int duration = -1;
 
-    public int age_limit = 0;
+    public int age_limit = -1;
     public int like_count = -1;
     public int dislike_count = -1;
     public String average_rating = "";

app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java

@@ -24,6 +24,8 @@ import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Vector;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Created by Christian Schabesberger on 06.08.15.
@@ -168,7 +170,8 @@ public class YoutubeStreamExtractor implements StreamExtractor {
     private static final String TAG = YoutubeStreamExtractor.class.toString();
     private final Document doc;
     private JSONObject playerArgs;
-    //private Map<String, String> videoInfoPage;
+    private boolean isAgeRestricted;
+    private Map<String, String> videoInfoPage;
 
     // static values
     private static final String DECRYPTION_FUNC_NAME="decrypt";
@@ -187,79 +190,123 @@ public class YoutubeStreamExtractor implements StreamExtractor {
         this.pageUrl = pageUrl;
         String pageContent = downloader.download(urlidhandler.cleanUrl(pageUrl));
         doc = Jsoup.parse(pageContent, pageUrl);
-        String ytPlayerConfigRaw;
         JSONObject ytPlayerConfig;
+        String playerUrl;
 
-        //attempt to load the youtube js player JSON arguments
+        // Check if the video is age restricted
-        boolean isLiveStream = false; //used to determine if this is a livestream or not
+        if (pageContent.contains("<meta property=\"og:restrictions:age")) {
+            String videoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
+                    urlidhandler.getVideoId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
+            String videoInfoPageString = downloader.download(videoInfoUrl);
+            videoInfoPage = Parser.compatParseMap(videoInfoPageString);
+            playerUrl = getPlayerUrlFromRestrictedVideo(pageUrl);
+            isAgeRestricted = true;
+        } else {
+            ytPlayerConfig = getPlayerConfig(pageContent);
+            playerArgs = getPlayerArgs(ytPlayerConfig);
+            playerUrl = getPlayerUrl(ytPlayerConfig);
+            isAgeRestricted = false;
+        }
+
+        if(decryptionCode.isEmpty()) {
+            decryptionCode = loadDecryptionCode(playerUrl);
+        }
+    }
+
+    private JSONObject getPlayerConfig(String pageContent) throws ParsingException {
         try {
-            ytPlayerConfigRaw =
+            String ytPlayerConfigRaw =
                     Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent);
-            ytPlayerConfig = new JSONObject(ytPlayerConfigRaw);
+            return new JSONObject(ytPlayerConfigRaw);
-            playerArgs = ytPlayerConfig.getJSONObject("args");
-
-            // check if we have a live stream. We need to filter it, since its not yet supported.
-            if((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
-                    || (playerArgs.get("url_encoded_fmt_stream_map").toString().isEmpty())) {
-                isLiveStream = true;
-            }
         } catch (Parser.RegexException e) {
             String errorReason = findErrorReason(doc);
             switch(errorReason) {
                 case "GEMA":
                     throw new GemaException(errorReason);
                 case "":
-                    throw new ParsingException("player config empty", e);
+                    throw new ContentNotAvailableException("Content not available: player config empty", e);
                 default:
                     throw new ContentNotAvailableException("Content not available", e);
             }
         } catch (JSONException e) {
             throw new ParsingException("Could not parse yt player config", e);
         }
+    }
+
+    private JSONObject getPlayerArgs(JSONObject playerConfig) throws ParsingException {
+        JSONObject playerArgs;
+
+        //attempt to load the youtube js player JSON arguments
+        boolean isLiveStream = false; //used to determine if this is a livestream or not
+        try {
+            playerArgs = playerConfig.getJSONObject("args");
+
+            // check if we have a live stream. We need to filter it, since its not yet supported.
+            if((playerArgs.has("ps") && playerArgs.get("ps").toString().equals("live"))
+                    || (playerArgs.get("url_encoded_fmt_stream_map").toString().isEmpty())) {
+                isLiveStream = true;
+            }
+        }  catch (JSONException e) {
+            throw new ParsingException("Could not parse yt player config", e);
+        }
         if (isLiveStream) {
             throw new LiveStreamException();
         }
 
+        return playerArgs;
+    }
 
-        /* not yet nececeary
+    private String getPlayerUrl(JSONObject playerConfig) throws ParsingException {
-
-
-        // get videoInfo page
         try {
-            //Parser.unescapeEntities(url_data_str, true).split("&")
+            // The Youtube service needs to be initialized by downloading the
-            String getVideoInfoUrl = GET_VIDEO_INFO_URL.replace("%%video_id%%",
+            // js-Youtube-player. This is done in order to get the algorithm
-                    urlidhandler.getVideoId(pageUrl)).replace("$$el_type$$", "&" + EL_INFO);
+            // for decrypting cryptic signatures inside certain stream urls.
-            videoInfoPage = Parser.compatParseMap(downloader.download(getVideoInfoUrl));
+            String playerUrl = "";
-        } catch(Exception e) {
-            throw new ParsingException("Could not load video info page.", e);
-        }
-        */
 
-        //----------------------------------
+            JSONObject ytAssets = playerConfig.getJSONObject("assets");
-        // load and parse description code, if it isn't already initialised
+            playerUrl = ytAssets.getString("js");
-        //----------------------------------
-        if (decryptionCode.isEmpty()) {
-            try {
-                // The Youtube service needs to be initialized by downloading the
-                // js-Youtube-player. This is done in order to get the algorithm
-                // for decrypting cryptic signatures inside certain stream urls.
-                JSONObject ytAssets = ytPlayerConfig.getJSONObject("assets");
-                String playerUrl = ytAssets.getString("js");
 
-                if (playerUrl.startsWith("//")) {
+            if (playerUrl.startsWith("//")) {
-                    playerUrl = "https:" + playerUrl;
+                playerUrl = "https:" + playerUrl;
-                }
-                decryptionCode = loadDecryptionCode(playerUrl);
-            } catch (JSONException e) {
-                throw new ParsingException(
-                        "Could not load decryption code for the Youtube service.", e);
             }
+            return playerUrl;
+        } catch (JSONException e) {
+            throw new ParsingException(
+                    "Could not load decryption code for the Youtube service.", e);
+        }
+    }
+
+    private String getPlayerUrlFromRestrictedVideo(String pageUrl) throws ParsingException {
+        try {
+            String playerUrl = "";
+            String videoId = urlidhandler.getVideoId(pageUrl);
+            String embedUrl = "https://www.youtube.com/embed/" + videoId;
+            String embedPageContent = downloader.download(embedUrl);
+            //todo: find out if this can be reapaced by Parser.matchGroup1()
+            Pattern assetsPattern = Pattern.compile("\"assets\":.+?\"js\":\\s*(\"[^\"]+\")");
+            Matcher patternMatcher = assetsPattern.matcher(embedPageContent);
+            while (patternMatcher.find()) {
+                playerUrl = patternMatcher.group(1);
+            }
+            playerUrl = playerUrl.replace("\\", "").replace("\"", "");
+
+            if (playerUrl.startsWith("//")) {
+                playerUrl = "https:" + playerUrl;
+            }
+            return playerUrl;
+        } catch (IOException e) {
+            throw new ParsingException(
+                    "Could load decryption code form restricted video for the Youtube service.", e);
         }
     }
 
     @Override
     public String getTitle() throws ParsingException {
-        try {//json player args method
+        try {
+            if (playerArgs == null) {
+                return videoInfoPage.get("title");
+            }
+            //json player args method
             return playerArgs.getString("title");
         } catch(JSONException je) {//html <meta> method
             je.printStackTrace();
@@ -283,7 +330,11 @@ public class YoutubeStreamExtractor implements StreamExtractor {
 
     @Override
     public String getUploader() throws ParsingException {
-        try {//json player args method
+        try {
+            if (playerArgs == null) {
+                return videoInfoPage.get("author");
+            }
+            //json player args method
             return playerArgs.getString("author");
         } catch(JSONException je) {
             je.printStackTrace();
@@ -299,6 +350,9 @@ public class YoutubeStreamExtractor implements StreamExtractor {
     @Override
     public int getLength() throws ParsingException {
         try {
+            if (playerArgs == null) {
+                return Integer.valueOf(videoInfoPage.get("length_seconds"));
+            }
             return playerArgs.getInt("length_seconds");
         } catch (JSONException e) {//todo: find fallback method
             throw new ParsingException("failed to load video duration from JSON args", e);
@@ -339,6 +393,9 @@ public class YoutubeStreamExtractor implements StreamExtractor {
         } catch (JSONException je) {
             throw new ParsingException(
                     "failed to extract thumbnail URL from JSON args; trying to extract it from HTML", je);
+        } catch (NullPointerException ne) {
+            // Get from the video info page instead
+            return videoInfoPage.get("thumbnail_url");
         }
     }
 
@@ -379,7 +436,13 @@ public class YoutubeStreamExtractor implements StreamExtractor {
     public List<VideoInfo.AudioStream> getAudioStreams() throws ParsingException {
         Vector<VideoInfo.AudioStream> audioStreams = new Vector<>();
         try{
-            String encoded_url_map = playerArgs.getString("adaptive_fmts");
+            String encoded_url_map;
+            // playerArgs could be null if the video is age restricted
+            if (playerArgs == null) {
+                encoded_url_map = videoInfoPage.get("adaptive_fmts");
+            } else {
+                encoded_url_map = playerArgs.getString("adaptive_fmts");
+            }
             for(String url_data_str : encoded_url_map.split(",")) {
                 // This loop iterates through multiple streams, therefor tags
                 // is related to one and the same stream at a time.
@@ -416,7 +479,13 @@ public class YoutubeStreamExtractor implements StreamExtractor {
         Vector<VideoInfo.VideoStream> videoStreams = new Vector<>();
 
         try{
-            String encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
+            String encoded_url_map;
+            // playerArgs could be null if the video is age restricted
+            if (playerArgs == null) {
+                encoded_url_map = videoInfoPage.get("url_encoded_fmt_stream_map");
+            } else {
+                encoded_url_map = playerArgs.getString("url_encoded_fmt_stream_map");
+            }
             for(String url_data_str : encoded_url_map.split(",")) {
                 try {
                     // This loop iterates through multiple streams, therefor tags
@@ -499,7 +568,8 @@ public class YoutubeStreamExtractor implements StreamExtractor {
                 int minutes = (minutesString.isEmpty() ? 0 : Integer.parseInt(minutesString));
                 int hours = (hoursString.isEmpty() ? 0 : Integer.parseInt(hoursString));
 
-                int ret = seconds + (60 * minutes) + (3600 * hours);//don't trust BODMAS!
+                //don't trust BODMAS!
+                int ret = seconds + (60 * minutes) + (3600 * hours);
                 //Log.d(TAG, "derived timestamp value:"+ret);
                 return ret;
                 //the ordering varies internationally
@@ -513,15 +583,24 @@ public class YoutubeStreamExtractor implements StreamExtractor {
 
     @Override
     public int getAgeLimit() throws ParsingException {
-        // Not yet implemented.
+        if (!isAgeRestricted) {
-        // Also you need to be logged in to see age restricted videos on youtube,
+            return 0;
-        // therefore NP is not able to receive such videos.
+        }
-        return 0;
+        try {
+            return Integer.valueOf(doc.head()
+                    .getElementsByAttributeValue("property", "og:restrictions:age")
+                    .attr("content").replace("+", ""));
+        } catch (Exception e) {
+            throw new ParsingException("Could not get age restriction");
+        }
     }
 
     @Override
     public String getAverageRating() throws ParsingException {
         try {
+            if (playerArgs == null) {
+                videoInfoPage.get("avg_rating");
+            }
             return playerArgs.getString("avg_rating");
         } catch (JSONException e) {
             throw new ParsingException("Could not get Average rating", e);

app/src/main/res/values/settings_keys.xml

@@ -3,6 +3,7 @@
     <!-- Categories -->
     <string name="settings_category_video_audio">settings_category_video_audio</string>
     <string name="settings_category_appearance">settings_category_appearance</string>
+    <string name="settings_content_options">settings_content_options</string>
     <string name="settings_category_other">settings_category_other</string>
     <!-- Key values -->
     <string name="download_path_key">download_path</string>
@@ -207,5 +208,6 @@
         <item>日本語</item>
         <item>한국어</item>
     </string-array>
+    <string name="show_age_restricted_content">show_age_restricted_content</string>
     <string name="use_tor_key">use_tor</string>
 </resources>

app/src/main/res/values/strings.xml

@@ -74,6 +74,9 @@
     <string name="background_player_playing_toast">Playing in background</string>
     <string name="c3s_url" translatable="false">https://www.c3s.cc/</string>
     <string name="play_btn_text">Play</string>
+    <string name="content">Content</string>
+    <string name="show_age_restricted_content_title">Show age restricted content</string>
+    <string name="video_is_age_restricted">Video is Age restricted. Enable age restricted videos in the settings first.</string>
 
     <!-- error strings -->
     <string name="general_error">Error</string>

app/src/main/res/xml/settings.xml

@@ -63,8 +63,8 @@
     </PreferenceCategory>
 
     <PreferenceCategory
-        android:key="@string/settings_category_other"
+        android:key="@string/settings_content_options"
-        android:title="@string/settings_category_other_title"
+        android:title="@string/content"
         android:textAllCaps="true">
 
         <ListPreference
@@ -75,6 +75,18 @@
             android:entryValues="@array/language_codes"
             android:defaultValue="@string/default_language_value" />
 
+        <CheckBoxPreference
+            android:key="@string/show_age_restricted_content"
+            android:title="@string/show_age_restricted_content_title"
+            android:defaultValue="false"/>
+
+    </PreferenceCategory>
+
+    <PreferenceCategory
+        android:key="@string/settings_category_other"
+        android:title="@string/settings_category_other_title"
+        android:textAllCaps="true">
+
         <EditTextPreference
             android:key="@string/download_path_key"
             android:title="@string/download_path_title"