Merge branch 'dev' into 1907renamePlaylist

README.md

@@ -13,7 +13,7 @@
 </p>
 <hr>
 <p align="center"><a href="#screenshots">Screenshots</a> &bull; <a href="#description">Description</a> &bull; <a href="#features">Features</a> &bull; <a href="#updates">Updates</a> &bull; <a href="#contribution">Contribution</a> &bull; <a href="#donate">Donate</a> &bull; <a href="#license">License</a></p>
-<p align="center"><a href="https://newpipe.schabi.org">Website</a> &bull; <a href="https://newpipe.schabi.org/blog/">Blog</a>  &bull; <a href="https://newpipe.schabi.org/press/">Press</a></p>
+<p align="center"><a href="https://newpipe.schabi.org">Website</a> &bull; <a href="https://newpipe.schabi.org/blog/">Blog</a> &bull; <a href="https://newpipe.schabi.org/FAQ/">FAQ</a> &bull; <a href="https://newpipe.schabi.org/press/">Press</a></p>
 <hr>
 
 <b>WARNING: THIS IS A BETA VERSION, THEREFORE YOU MAY ENCOUNTER BUGS. IF YOU DO, OPEN AN ISSUE VIA OUR GITHUB REPOSITORY.</b>

app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java

@@ -832,7 +832,6 @@ public class DownloadDialog extends DialogFragment implements RadioGroup.OnCheck
                     psArgs = new String[]{
                             selectedStream.getFormat().getSuffix(),
                             "false",// ignore empty frames
-                            "false",// detect youtube duplicate lines
                     };
                 }
                 break;

app/src/main/java/org/schabi/newpipe/streams/SrtFromTtmlWriter.java

@@ -0,0 +1,95 @@
+package org.schabi.newpipe.streams;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.nodes.Node;
+import org.jsoup.nodes.TextNode;
+import org.jsoup.parser.Parser;
+import org.jsoup.select.Elements;
+import org.schabi.newpipe.streams.io.SharpStream;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author kapodamy
+ */
+public class SrtFromTtmlWriter {
+    private static final String NEW_LINE = "\r\n";
+
+    private SharpStream out;
+    private boolean ignoreEmptyFrames;
+    private final Charset charset = StandardCharsets.UTF_8;
+
+    private int frameIndex = 0;
+
+    public SrtFromTtmlWriter(SharpStream out, boolean ignoreEmptyFrames) {
+        this.out = out;
+        this.ignoreEmptyFrames = ignoreEmptyFrames;
+    }
+
+    private static String getTimestamp(Element frame, String attr) {
+        return frame
+                .attr(attr)
+                .replace('.', ',');// SRT subtitles uses comma as decimal separator
+    }
+
+    private void writeFrame(String begin, String end, StringBuilder text) throws IOException {
+        writeString(String.valueOf(frameIndex++));
+        writeString(NEW_LINE);
+        writeString(begin);
+        writeString(" --> ");
+        writeString(end);
+        writeString(NEW_LINE);
+        writeString(text.toString());
+        writeString(NEW_LINE);
+        writeString(NEW_LINE);
+    }
+
+    private void writeString(String text) throws IOException {
+        out.write(text.getBytes(charset));
+    }
+
+    public void build(SharpStream ttml) throws IOException {
+        /*
+         * TTML parser with BASIC support
+         * multiple CUE is not supported
+         * styling is not supported
+         * tag timestamps (in auto-generated subtitles) are not supported, maybe in the future
+         * also TimestampTagOption enum is not applicable
+         * Language parsing is not supported
+         */
+
+        // parse XML
+        byte[] buffer = new byte[(int) ttml.available()];
+        ttml.read(buffer);
+        Document doc = Jsoup.parse(new ByteArrayInputStream(buffer), "UTF-8", "", Parser.xmlParser());
+
+        StringBuilder text = new StringBuilder(128);
+        Elements paragraph_list = doc.select("body > div > p");
+
+        // check if has frames
+        if (paragraph_list.size() < 1) return;
+
+        for (Element paragraph : paragraph_list) {
+            text.setLength(0);
+
+            for (Node children : paragraph.childNodes()) {
+                if (children instanceof TextNode)
+                    text.append(((TextNode) children).text());
+                else if (children instanceof Element && ((Element) children).tagName().equalsIgnoreCase("br"))
+                    text.append(NEW_LINE);
+            }
+
+            if (ignoreEmptyFrames && text.length() < 1) continue;
+
+            String begin = getTimestamp(paragraph, "begin");
+            String end = getTimestamp(paragraph, "end");
+
+            writeFrame(begin, end, text);
+        }
+    }
+}

app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java

@@ -1,369 +0,0 @@
-package org.schabi.newpipe.streams;
-
-import org.schabi.newpipe.streams.io.SharpStream;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import org.xml.sax.SAXException;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.text.ParseException;
-import java.util.Locale;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.xpath.XPathExpressionException;
-
-/**
- * @author kapodamy
- */
-public class SubtitleConverter {
-    private static final String NEW_LINE = "\r\n";
-
-    public void dumpTTML(SharpStream in, final SharpStream out, final boolean ignoreEmptyFrames, final boolean detectYoutubeDuplicateLines
-    ) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException {
-
-        final FrameWriter callback = new FrameWriter() {
-            int frameIndex = 0;
-            final Charset charset = Charset.forName("utf-8");
-
-            @Override
-            public void yield(SubtitleFrame frame) throws IOException {
-                if (ignoreEmptyFrames && frame.isEmptyText()) {
-                    return;
-                }
-                out.write(String.valueOf(frameIndex++).getBytes(charset));
-                out.write(NEW_LINE.getBytes(charset));
-                out.write(getTime(frame.start, true).getBytes(charset));
-                out.write(" --> ".getBytes(charset));
-                out.write(getTime(frame.end, true).getBytes(charset));
-                out.write(NEW_LINE.getBytes(charset));
-                out.write(frame.text.getBytes(charset));
-                out.write(NEW_LINE.getBytes(charset));
-                out.write(NEW_LINE.getBytes(charset));
-            }
-        };
-
-        read_xml_based(in, callback, detectYoutubeDuplicateLines,
-                "tt", "xmlns", "http://www.w3.org/ns/ttml",
-                new String[]{"timedtext", "head", "wp"},
-                new String[]{"body", "div", "p"},
-                "begin", "end", true
-        );
-    }
-
-    private void read_xml_based(SharpStream source, FrameWriter callback, boolean detectYoutubeDuplicateLines,
-                                String root, String formatAttr, String formatVersion, String[] cuePath, String[] framePath,
-                                String timeAttr, String durationAttr, boolean hasTimestamp
-    ) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException {
-        /*
-         * XML based subtitles parser with BASIC support
-         * multiple CUE is not supported
-         * styling is not supported
-         * tag timestamps (in auto-generated subtitles) are not supported, maybe in the future
-         * also TimestampTagOption enum is not applicable
-         * Language parsing is not supported
-         */
-
-        byte[] buffer = new byte[(int) source.available()];
-        source.read(buffer);
-
-        DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
-        factory.setNamespaceAware(true);
-        DocumentBuilder builder = factory.newDocumentBuilder();
-        Document xml = builder.parse(new ByteArrayInputStream(buffer));
-
-        String attr;
-
-        // get the format version or namespace
-        Element node = xml.getDocumentElement();
-
-        if (node == null) {
-            throw new ParseException("Can't get the format version. ¿wrong namespace?", -1);
-        } else if (!node.getNodeName().equals(root)) {
-            throw new ParseException("Invalid root", -1);
-        }
-
-        if (formatAttr.equals("xmlns")) {
-            if (!node.getNamespaceURI().equals(formatVersion)) {
-                throw new UnsupportedOperationException("Expected xml namespace: " + formatVersion);
-            }
-        } else {
-            attr = node.getAttributeNS(formatVersion, formatAttr);
-            if (attr == null) {
-                throw new ParseException("Can't get the format attribute", -1);
-            }
-            if (!attr.equals(formatVersion)) {
-                throw new ParseException("Invalid format version : " + attr, -1);
-            }
-        }
-
-        NodeList node_list;
-
-        int line_break = 0;// Maximum characters per line if present (valid for TranScript v3)
-
-        if (!hasTimestamp) {
-            node_list = selectNodes(xml, cuePath, formatVersion);
-
-            if (node_list != null) {
-                // if the subtitle has multiple CUEs, use the highest value
-                for (int i = 0; i < node_list.getLength(); i++) {
-                    try {
-                        int tmp = Integer.parseInt(((Element) node_list.item(i)).getAttributeNS(formatVersion, "ah"));
-                        if (tmp > line_break) {
-                            line_break = tmp;
-                        }
-                    } catch (Exception err) {
-                    }
-                }
-            }
-        }
-
-        // parse every frame
-        node_list = selectNodes(xml, framePath, formatVersion);
-
-        if (node_list == null) {
-            return;// no frames detected
-        }
-
-        int fs_ff = -1;// first timestamp of first frame
-        boolean limit_lines = false;
-
-        for (int i = 0; i < node_list.getLength(); i++) {
-            Element elem = (Element) node_list.item(i);
-            SubtitleFrame obj = new SubtitleFrame();
-            obj.text = elem.getTextContent();
-
-            attr = elem.getAttribute(timeAttr);// ¡this cant be null!
-            obj.start = hasTimestamp ? parseTimestamp(attr) : Integer.parseInt(attr);
-
-            attr = elem.getAttribute(durationAttr);
-            if (obj.text == null || attr == null) {
-                continue;// normally is a blank line (on auto-generated subtitles) ignore
-            }
-
-            if (hasTimestamp) {
-                obj.end = parseTimestamp(attr);
-
-                if (detectYoutubeDuplicateLines) {
-                    if (limit_lines) {
-                        int swap = obj.end;
-                        obj.end = fs_ff;
-                        fs_ff = swap;
-                    } else {
-                        if (fs_ff < 0) {
-                            fs_ff = obj.end;
-                        } else {
-                            if (fs_ff < obj.start) {
-                                limit_lines = true;// the subtitles has duplicated lines
-                            } else {
-                                detectYoutubeDuplicateLines = false;
-                            }
-                        }
-                    }
-                }
-            } else {
-                obj.end = obj.start + Integer.parseInt(attr);
-            }
-
-            if (/*node.getAttribute("w").equals("1") &&*/line_break > 1 && obj.text.length() > line_break) {
-
-                // implement auto line breaking (once)
-                StringBuilder text = new StringBuilder(obj.text);
-                obj.text = null;
-
-                switch (text.charAt(line_break)) {
-                    case ' ':
-                    case '\t':
-                        putBreakAt(line_break, text);
-                        break;
-                    default:// find the word start position
-                        for (int j = line_break - 1; j > 0; j--) {
-                            switch (text.charAt(j)) {
-                                case ' ':
-                                case '\t':
-                                    putBreakAt(j, text);
-                                    j = -1;
-                                    break;
-                                case '\r':
-                                case '\n':
-                                    j = -1;// long word, just ignore
-                                    break;
-                            }
-                        }
-                        break;
-                }
-
-                obj.text = text.toString();// set the processed text
-            }
-
-            callback.yield(obj);
-        }
-    }
-
-    private static NodeList selectNodes(Document xml, String[] path, String namespaceUri) {
-        Element ref = xml.getDocumentElement();
-
-        for (int i = 0; i < path.length - 1; i++) {
-            NodeList nodes = ref.getChildNodes();
-            if (nodes.getLength() < 1) {
-                return null;
-            }
-
-            Element elem;
-            for (int j = 0; j < nodes.getLength(); j++) {
-                if (nodes.item(j).getNodeType() == Node.ELEMENT_NODE) {
-                    elem = (Element) nodes.item(j);
-                    if (elem.getNodeName().equals(path[i]) && elem.getNamespaceURI().equals(namespaceUri)) {
-                        ref = elem;
-                        break;
-                    }
-                }
-            }
-        }
-
-        return ref.getElementsByTagNameNS(namespaceUri, path[path.length - 1]);
-    }
-
-    private static int parseTimestamp(String multiImpl) throws NumberFormatException, ParseException {
-        if (multiImpl.length() < 1) {
-            return 0;
-        } else if (multiImpl.length() == 1) {
-            return Integer.parseInt(multiImpl) * 1000;// ¡this must be a number in seconds!
-        }
-
-        // detect wallclock-time
-        if (multiImpl.startsWith("wallclock(")) {
-            throw new UnsupportedOperationException("Parsing wallclock timestamp is not implemented");
-        }
-
-        // detect offset-time
-        if (multiImpl.indexOf(':') < 0) {
-            int multiplier = 1000;
-            char metric = multiImpl.charAt(multiImpl.length() - 1);
-            switch (metric) {
-                case 'h':
-                    multiplier *= 3600000;
-                    break;
-                case 'm':
-                    multiplier *= 60000;
-                    break;
-                case 's':
-                    if (multiImpl.charAt(multiImpl.length() - 2) == 'm') {
-                        multiplier = 1;// ms
-                    }
-                    break;
-                default:
-                    if (!Character.isDigit(metric)) {
-                        throw new NumberFormatException("Invalid metric suffix found on : " + multiImpl);
-                    }
-                    metric = '\0';
-                    break;
-            }
-            try {
-                String offset_time = multiImpl;
-
-                if (multiplier == 1) {
-                    offset_time = offset_time.substring(0, offset_time.length() - 2);
-                } else if (metric != '\0') {
-                    offset_time = offset_time.substring(0, offset_time.length() - 1);
-                }
-
-                double time_metric_based = Double.parseDouble(offset_time);
-                if (Math.abs(time_metric_based) <= Double.MAX_VALUE) {
-                    return (int) (time_metric_based * multiplier);
-                }
-            } catch (Exception err) {
-                throw new UnsupportedOperationException("Invalid or not implemented timestamp on: " + multiImpl);
-            }
-        }
-
-        // detect clock-time
-        int time = 0;
-        String[] units = multiImpl.split(":");
-
-        if (units.length < 3) {
-            throw new ParseException("Invalid clock-time timestamp", -1);
-        }
-
-        time += Integer.parseInt(units[0]) * 3600000;// hours
-        time += Integer.parseInt(units[1]) * 60000;//minutes
-        time += Float.parseFloat(units[2]) * 1000f;// seconds and milliseconds (if present)
-
-        // frames and sub-frames are ignored (not implemented)
-        // time += units[3] * fps;
-        return time;
-    }
-
-    private static void putBreakAt(int idx, StringBuilder str) {
-        // this should be optimized at compile time
-
-        if (NEW_LINE.length() > 1) {
-            str.delete(idx, idx + 1);// remove after replace
-            str.insert(idx, NEW_LINE);
-        } else {
-            str.setCharAt(idx, NEW_LINE.charAt(0));
-        }
-    }
-
-    private static String getTime(int time, boolean comma) {
-        // cast every value to integer to avoid auto-round in ToString("00").
-        StringBuilder str = new StringBuilder(12);
-        str.append(numberToString(time / 1000 / 3600, 2));// hours
-        str.append(':');
-        str.append(numberToString(time / 1000 / 60 % 60, 2));// minutes
-        str.append(':');
-        str.append(numberToString(time / 1000 % 60, 2));// seconds
-        str.append(comma ? ',' : '.');
-        str.append(numberToString(time % 1000, 3));// miliseconds
-
-        return str.toString();
-    }
-
-    private static String numberToString(int nro, int pad) {
-        return String.format(Locale.ENGLISH, "%0".concat(String.valueOf(pad)).concat("d"), nro);
-    }
-
-
-    /******************
-     * helper classes *
-     ******************/
-
-    private interface FrameWriter {
-
-        void yield(SubtitleFrame frame) throws IOException;
-    }
-
-    private static class SubtitleFrame {
-        //Java no support unsigned int
-
-        public int end;
-        public int start;
-        public String text = "";
-
-        private boolean isEmptyText() {
-            if (text == null) {
-                return true;
-            }
-
-            for (int i = 0; i < text.length(); i++) {
-                switch (text.charAt(i)) {
-                    case ' ':
-                    case '\t':
-                    case '\r':
-                    case '\n':
-                        break;
-                    default:
-                        return false;
-                }
-            }
-
-            return true;
-        }
-    }
-
-}

app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java

@@ -80,7 +80,7 @@ public abstract class Postprocessing implements Serializable {
 
     private transient DownloadMission mission;
 
-    private File tempFile;
+    private transient File tempFile;
 
     Postprocessing(boolean reserveSpace, boolean worksOnSameFile, String algorithmName) {
         this.reserveSpace = reserveSpace;
@@ -95,8 +95,12 @@ public abstract class Postprocessing implements Serializable {
 
     public void cleanupTemporalDir() {
         if (tempFile != null && tempFile.exists()) {
-            //noinspection ResultOfMethodCallIgnored
+            try {
-            tempFile.delete();
+                //noinspection ResultOfMethodCallIgnored
+                tempFile.delete();
+            } catch (Exception e) {
+                // nothing to do
+            }
         }
     }
 

app/src/main/java/us/shandian/giga/postprocessing/TtmlConverter.java

@@ -2,15 +2,10 @@ package us.shandian.giga.postprocessing;
 
 import android.util.Log;
 
-import org.schabi.newpipe.streams.SubtitleConverter;
+import org.schabi.newpipe.streams.SrtFromTtmlWriter;
 import org.schabi.newpipe.streams.io.SharpStream;
-import org.xml.sax.SAXException;
 
 import java.io.IOException;
-import java.text.ParseException;
-
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.xpath.XPathExpressionException;
 
 /**
  * @author kapodamy
@@ -27,33 +22,16 @@ class TtmlConverter extends Postprocessing {
     int process(SharpStream out, SharpStream... sources) throws IOException {
         // check if the subtitle is already in srt and copy, this should never happen
         String format = getArgumentAt(0, null);
+        boolean ignoreEmptyFrames = getArgumentAt(1, "true").equals("true");
 
         if (format == null || format.equals("ttml")) {
-            SubtitleConverter ttmlDumper = new SubtitleConverter();
+            SrtFromTtmlWriter writer = new SrtFromTtmlWriter(out, ignoreEmptyFrames);
 
             try {
-                ttmlDumper.dumpTTML(
+                writer.build(sources[0]);
-                        sources[0],
-                        out,
-                        getArgumentAt(1, "true").equals("true"),
-                        getArgumentAt(2, "true").equals("true")
-                );
             } catch (Exception err) {
                 Log.e(TAG, "subtitle parse failed", err);
-
+                return err instanceof IOException ? 1 : 8;
-                if (err instanceof IOException) {
-                    return 1;
-                } else if (err instanceof ParseException) {
-                    return 2;
-                } else if (err instanceof SAXException) {
-                    return 3;
-                } else if (err instanceof ParserConfigurationException) {
-                    return 4;
-                } else if (err instanceof XPathExpressionException) {
-                    return 7;
-                }
-
-                return 8;
             }
 
             return OK_RESULT;

app/src/main/java/us/shandian/giga/service/DownloadManager.java

@@ -139,6 +139,9 @@ public class DownloadManager {
             Log.d(TAG, "Loading pending downloads from directory: " + mPendingMissionsDir.getAbsolutePath());
         }
 
+        File tempDir = pickAvailableTemporalDir(ctx);
+        Log.i(TAG, "using '" + tempDir + "' as temporal directory");
+
         for (File sub : subs) {
             if (!sub.isFile()) continue;
             if (sub.getName().equals(".tmp")) continue;
@@ -184,7 +187,7 @@ public class DownloadManager {
 
             if (mis.psAlgorithm != null) {
                 mis.psAlgorithm.cleanupTemporalDir();
-                mis.psAlgorithm.setTemporalDir(pickAvailableTemporalDir(ctx));
+                mis.psAlgorithm.setTemporalDir(tempDir);
             }
 
             mis.metadata = sub;
@@ -513,13 +516,21 @@ public class DownloadManager {
     }
 
     static File pickAvailableTemporalDir(@NonNull Context ctx) {
-        if (isDirectoryAvailable(ctx.getExternalFilesDir(null)))
+        File dir = ctx.getExternalFilesDir(null);
-            return ctx.getExternalFilesDir(null);
+        if (isDirectoryAvailable(dir)) return dir;
-        else if (isDirectoryAvailable(ctx.getFilesDir()))
+
-            return ctx.getFilesDir();
+        dir = ctx.getFilesDir();
+        if (isDirectoryAvailable(dir)) return dir;
 
         // this never should happen
-        return ctx.getDir("tmp", Context.MODE_PRIVATE);
+        dir = ctx.getDir("muxing_tmp", Context.MODE_PRIVATE);
+        if (isDirectoryAvailable(dir)) return dir;
+
+        // fallback to cache dir
+        dir = ctx.getCacheDir();
+        if (isDirectoryAvailable(dir)) return dir;
+
+        throw new RuntimeException("Not temporal directories are available");
     }
 
     @Nullable