From 160a33e8c844a7aa7534fe798a3472a015315480 Mon Sep 17 00:00:00 2001 From: kapodamy Date: Mon, 30 Sep 2019 23:52:49 -0300 Subject: [PATCH] misc changes * OggFromWebMWriter: rewrite (again), reduce iterations over the input. Works as-is (video streams are not supported) * WebMReader: use int for SimpleBlock.dataSize instead of long * Download Recovery: allow recovering uninitialized downloads * check range-requests using HEAD method instead of GET * DownloadRunnableFallback: add workaround for 32kB/s issue, unknown issue origin, wont fix * reporting downloads errors now include the source url with the selected quality and format --- .../newpipe/streams/OggFromWebMWriter.java | 216 +++++------------- .../schabi/newpipe/streams/WebMReader.java | 4 +- .../giga/get/DownloadInitializer.java | 35 +-- .../us/shandian/giga/get/DownloadMission.java | 36 +-- .../giga/get/DownloadMissionRecover.java | 146 +++++++++--- .../shandian/giga/get/DownloadRunnable.java | 2 +- .../giga/get/DownloadRunnableFallback.java | 20 +- .../giga/get/MissionRecoveryInfo.java | 43 +++- .../giga/ui/adapter/MissionAdapter.java | 36 ++- app/src/main/res/values-es/strings.xml | 2 +- app/src/main/res/values/strings.xml | 2 +- 11 files changed, 294 insertions(+), 248 deletions(-) diff --git a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java index 091ae6d2a..e6363e423 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java +++ b/app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java @@ -12,8 +12,6 @@ import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.util.ArrayList; -import java.util.Random; import javax.annotation.Nullable; @@ -23,15 +21,13 @@ import javax.annotation.Nullable; public class OggFromWebMWriter implements Closeable { private static final byte FLAG_UNSET = 0x00; - private static final byte FLAG_CONTINUED = 0x01; + //private static final byte FLAG_CONTINUED = 0x01; private static final byte FLAG_FIRST = 0x02; private static final byte FLAG_LAST = 0x04; private final static byte HEADER_CHECKSUM_OFFSET = 22; private final static byte HEADER_SIZE = 27; - private final static short BUFFER_SIZE = 8 * 1024;// 8KiB - private final static int TIME_SCALE_NS = 1000000000; private boolean done = false; @@ -43,7 +39,6 @@ public class OggFromWebMWriter implements Closeable { private int sequence_count = 0; private final int STREAM_ID; private byte packet_flag = FLAG_FIRST; - private int track_index = 0; private WebMReader webm = null; private WebMTrack webm_track = null; @@ -71,7 +66,7 @@ public class OggFromWebMWriter implements Closeable { this.source = source; this.output = target; - this.STREAM_ID = (new Random(System.currentTimeMillis())).nextInt(); + this.STREAM_ID = (int) System.currentTimeMillis(); populate_crc32_table(); } @@ -130,7 +125,6 @@ public class OggFromWebMWriter implements Closeable { try { webm_track = webm.selectTrack(trackIndex); - track_index = trackIndex; } finally { parsed = true; } @@ -154,8 +148,11 @@ public class OggFromWebMWriter implements Closeable { public void build() throws IOException { float resolution; - int read; - byte[] buffer; + SimpleBlock bloq; + ByteBuffer header = ByteBuffer.allocate(27 + (255 * 255)); + ByteBuffer page = ByteBuffer.allocate(64 * 1024); + + header.order(ByteOrder.LITTLE_ENDIAN); /* step 1: get the amount of frames per seconds */ switch (webm_track.kind) { @@ -176,57 +173,32 @@ public class OggFromWebMWriter implements Closeable { throw new RuntimeException("not implemented"); } - /* step 2a: create packet with code init data */ - ArrayList data_extra = new ArrayList<>(4); - + /* step 2: create packet with code init data */ if (webm_track.codecPrivate != null) { addPacketSegment(webm_track.codecPrivate.length); - ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + webm_track.codecPrivate.length); - - make_packetHeader(0x00, buff, webm_track.codecPrivate); - data_extra.add(buff.array()); + make_packetHeader(0x00, header, webm_track.codecPrivate); + write(header); + output.write(webm_track.codecPrivate); } - /* step 2b: create packet with metadata */ - buffer = make_metadata(); + /* step 3: create packet with metadata */ + byte[] buffer = make_metadata(); if (buffer != null) { addPacketSegment(buffer.length); - ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + buffer.length); - - make_packetHeader(0x00, buff, buffer); - data_extra.add(buff.array()); + make_packetHeader(0x00, header, buffer); + write(header); + output.write(buffer); } - - /* step 3: calculate amount of packets */ - SimpleBlock bloq; - int reserve_header = 0; - int headers_amount = 0; - + /* step 4: calculate amount of packets */ while (webm_segment != null) { bloq = getNextBlock(); - if (addPacketSegment(bloq)) { - continue; - } - - reserve_header += HEADER_SIZE + segment_table_size;// header size - clearSegmentTable(); - webm_block = bloq; - headers_amount++; - } - - /* step 4: create packet headers */ - rewind_source(); - - ByteBuffer headers = byte_buffer(reserve_header); - short[] headers_size = new short[headers_amount]; - int header_index = 0; - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (addPacketSegment(bloq)) { + if (bloq != null && addPacketSegment(bloq)) { + int pos = page.position(); + //noinspection ResultOfMethodCallIgnored + bloq.data.read(page.array(), pos, bloq.dataSize); + page.position(pos + bloq.dataSize); continue; } @@ -251,75 +223,21 @@ public class OggFromWebMWriter implements Closeable { elapsed_ns = elapsed_ns / TIME_SCALE_NS; elapsed_ns = Math.ceil(elapsed_ns * resolution); - // create header - headers_size[header_index++] = make_packetHeader((long) elapsed_ns, headers, null); + // create header and calculate page checksum + int checksum = make_packetHeader((long) elapsed_ns, header, null); + checksum = calc_crc32(checksum, page.array(), page.position()); + + header.putInt(HEADER_CHECKSUM_OFFSET, checksum); + + // dump data + write(header); + write(page); + webm_block = bloq; } - - - /* step 5: calculate checksums */ - rewind_source(); - - int offset = 0; - buffer = new byte[BUFFER_SIZE]; - - for (header_index = 0; header_index < headers_size.length; header_index++) { - int checksum_offset = offset + HEADER_CHECKSUM_OFFSET; - int checksum = headers.getInt(checksum_offset); - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (!addPacketSegment(bloq)) { - clearSegmentTable(); - webm_block = bloq; - break; - } - - // calculate page checksum - while ((read = bloq.data.read(buffer)) > 0) { - checksum = calc_crc32(checksum, buffer, 0, read); - } - } - - headers.putInt(checksum_offset, checksum); - offset += headers_size[header_index]; - } - - /* step 6: write extra headers */ - rewind_source(); - - for (byte[] buff : data_extra) { - output.write(buff); - } - - /* step 7: write stream packets */ - byte[] headers_buffers = headers.array(); - offset = 0; - buffer = new byte[BUFFER_SIZE]; - - for (header_index = 0; header_index < headers_size.length; header_index++) { - output.write(headers_buffers, offset, headers_size[header_index]); - offset += headers_size[header_index]; - - while (webm_segment != null) { - bloq = getNextBlock(); - - if (addPacketSegment(bloq)) { - while ((read = bloq.data.read(buffer)) > 0) { - output.write(buffer, 0, read); - } - } else { - clearSegmentTable(); - webm_block = bloq; - break; - } - } - } } - private short make_packetHeader(long gran_pos, ByteBuffer buffer, byte[] immediate_page) { - int offset = buffer.position(); + private int make_packetHeader(long gran_pos, @NonNull ByteBuffer buffer, byte[] immediate_page) { short length = HEADER_SIZE; buffer.putInt(0x5367674f);// "OggS" binary string in little-endian @@ -340,17 +258,15 @@ public class OggFromWebMWriter implements Closeable { clearSegmentTable();// clear segment table for next header - int checksum_crc32 = calc_crc32(0x00, buffer.array(), offset, length); + int checksum_crc32 = calc_crc32(0x00, buffer.array(), length); if (immediate_page != null) { - checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, 0, immediate_page.length); - System.arraycopy(immediate_page, 0, buffer.array(), length, immediate_page.length); + checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length); + buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32); segment_table_next_timestamp -= TIME_SCALE_NS; } - buffer.putInt(offset + HEADER_CHECKSUM_OFFSET, checksum_crc32); - - return length; + return checksum_crc32; } @Nullable @@ -358,7 +274,7 @@ public class OggFromWebMWriter implements Closeable { if ("A_OPUS".equals(webm_track.codecId)) { return new byte[]{ 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string - 0x07, 0x00, 0x00, 0x00,// writing application string size + 0x07, 0x00, 0x00, 0x00,// writting application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags) }; @@ -366,7 +282,7 @@ public class OggFromWebMWriter implements Closeable { return new byte[]{ 0x03,// ???????? 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string - 0x07, 0x00, 0x00, 0x00,// writing application string size + 0x07, 0x00, 0x00, 0x00,// writting application string size 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags) @@ -387,22 +303,9 @@ public class OggFromWebMWriter implements Closeable { return null; } - private void rewind_source() throws IOException { - source.rewind(); - - webm = new WebMReader(source); - webm.parse(); - webm_track = webm.selectTrack(track_index); - webm_segment = webm.getNextSegment(); - webm_cluster = null; - webm_block = null; - webm_block_last_timecode = 0L; - - segment_table_next_timestamp = TIME_SCALE_NS; - } - - private ByteBuffer byte_buffer(int size) { - return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN); + private void write(ByteBuffer buffer) throws IOException { + output.write(buffer.array(), 0, buffer.position()); + buffer.position(0); } // @@ -460,41 +363,32 @@ public class OggFromWebMWriter implements Closeable { // private void clearSegmentTable() { - if (packet_flag != FLAG_CONTINUED) { - segment_table_next_timestamp += TIME_SCALE_NS; - packet_flag = FLAG_UNSET; - } + segment_table_next_timestamp += TIME_SCALE_NS; + packet_flag = FLAG_UNSET; segment_table_size = 0; } private boolean addPacketSegment(SimpleBlock block) { - if (block == null) { - return false; - } - long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay; if (timestamp >= segment_table_next_timestamp) { return false; } - boolean result = addPacketSegment((int) block.dataSize); - - if (!result && segment_table_next_timestamp < timestamp) { - // WARNING: ¡¡¡¡ not implemented (lack of documentation) !!!! - packet_flag = FLAG_CONTINUED; - } - - return result; + return addPacketSegment(block.dataSize); } private boolean addPacketSegment(int size) { + if (size > 65025) { + throw new UnsupportedOperationException("page size cannot be larger than 65025"); + } + int available = (segment_table.length - segment_table_size) * 255; - boolean extra = size == 255; + boolean extra = (size % 255) == 0; if (extra) { // add a zero byte entry in the table - // required to indicate the sample size is exactly 255 + // required to indicate the sample size is multiple of 255 available -= 255; } @@ -528,12 +422,10 @@ public class OggFromWebMWriter implements Closeable { } } - private int calc_crc32(int initial_crc, byte[] buffer, int offset, int size) { - size += offset; - - for (; offset < size; offset++) { + private int calc_crc32(int initial_crc, byte[] buffer, int size) { + for (int i = 0; i < size; i++) { int reg = (initial_crc >>> 24) & 0xff; - initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[offset] & 0xff)]; + initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)]; } return initial_crc; diff --git a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java index 13c15370d..4cb96d901 100644 --- a/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java +++ b/app/src/main/java/org/schabi/newpipe/streams/WebMReader.java @@ -368,7 +368,7 @@ public class WebMReader { obj.trackNumber = readEncodedNumber(); obj.relativeTimeCode = stream.readShort(); obj.flags = (byte) stream.read(); - obj.dataSize = (ref.offset + ref.size) - stream.position(); + obj.dataSize = (int) ((ref.offset + ref.size) - stream.position()); obj.createdFromBlock = ref.type == ID_Block; // NOTE: lacing is not implemented, and will be mixed with the stream data @@ -465,7 +465,7 @@ public class WebMReader { public short relativeTimeCode; public long absoluteTimeCodeNs; public byte flags; - public long dataSize; + public int dataSize; private final Element ref; public boolean isKeyframe() { diff --git a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java index 593feafa7..17a2a7403 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadInitializer.java @@ -14,6 +14,7 @@ import java.nio.channels.ClosedByInterruptException; import us.shandian.giga.util.Utility; import static org.schabi.newpipe.BuildConfig.DEBUG; +import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN; public class DownloadInitializer extends Thread { private final static String TAG = "DownloadInitializer"; @@ -29,9 +30,9 @@ public class DownloadInitializer extends Thread { mConn = null; } - private static void safeClose(HttpURLConnection con) { + private void dispose() { try { - con.getInputStream().close(); + mConn.getInputStream().close(); } catch (Exception e) { // nothing to do } @@ -52,9 +53,9 @@ public class DownloadInitializer extends Thread { long lowestSize = Long.MAX_VALUE; for (int i = 0; i < mMission.urls.length && mMission.running; i++) { - mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1); + mConn = mMission.openConnection(mMission.urls[i], true, -1, -1); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (Thread.interrupted()) return; long length = Utility.getContentLength(mConn); @@ -82,9 +83,9 @@ public class DownloadInitializer extends Thread { } } else { // ask for the current resource length - mConn = mMission.openConnection(mId, -1, -1); + mConn = mMission.openConnection(true, -1, -1); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (!mMission.running || Thread.interrupted()) return; @@ -108,9 +109,9 @@ public class DownloadInitializer extends Thread { } } else { // Open again - mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length); + mConn = mMission.openConnection(true, mMission.length - 10, mMission.length); mMission.establishConnection(mId, mConn); - safeClose(mConn); + dispose(); if (!mMission.running || Thread.interrupted()) return; @@ -171,7 +172,14 @@ public class DownloadInitializer extends Thread { } catch (InterruptedIOException | ClosedByInterruptException e) { return; } catch (Exception e) { - if (!mMission.running) return; + if (!mMission.running || super.isInterrupted()) return; + + if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { + // for youtube streams. The url has expired + interrupt(); + mMission.doRecover(e); + return; + } if (e instanceof IOException && e.getMessage().contains("Permission denied")) { mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e); @@ -194,13 +202,6 @@ public class DownloadInitializer extends Thread { @Override public void interrupt() { super.interrupt(); - - if (mConn != null) { - try { - mConn.disconnect(); - } catch (Exception e) { - // nothing to do - } - } + if (mConn != null) dispose(); } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMission.java b/app/src/main/java/us/shandian/giga/get/DownloadMission.java index 77b417118..918d6dbea 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMission.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMission.java @@ -204,22 +204,24 @@ public class DownloadMission extends Mission { /** * Opens a connection * - * @param threadId id of the calling thread, used only for debugging - * @param rangeStart range start - * @param rangeEnd range end + * @param headRequest {@code true} for use {@code HEAD} request method, otherwise, {@code GET} is used + * @param rangeStart range start + * @param rangeEnd range end * @return a {@link java.net.URLConnection URLConnection} linking to the URL. * @throws IOException if an I/O exception occurs. */ - HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException { - return openConnection(urls[current], threadId, rangeStart, rangeEnd); + HttpURLConnection openConnection(boolean headRequest, long rangeStart, long rangeEnd) throws IOException { + return openConnection(urls[current], headRequest, rangeStart, rangeEnd); } - HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException { + HttpURLConnection openConnection(String url, boolean headRequest, long rangeStart, long rangeEnd) throws IOException { HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); conn.setInstanceFollowRedirects(true); conn.setRequestProperty("User-Agent", DownloaderImpl.USER_AGENT); conn.setRequestProperty("Accept", "*/*"); + if (headRequest) conn.setRequestMethod("HEAD"); + // BUG workaround: switching between networks can freeze the download forever conn.setConnectTimeout(30000); conn.setReadTimeout(10000); @@ -229,10 +231,6 @@ public class DownloadMission extends Mission { if (rangeEnd > 0) req += rangeEnd; conn.setRequestProperty("Range", req); - - if (DEBUG) { - Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range")); - } } return conn; @@ -245,13 +243,14 @@ public class DownloadMission extends Mission { * @throws HttpError if the HTTP Status-Code is not satisfiable */ void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError { - conn.connect(); int statusCode = conn.getResponseCode(); if (DEBUG) { + Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range")); Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode); } + switch (statusCode) { case 204: case 205: @@ -676,6 +675,15 @@ public class DownloadMission extends Mission { return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished(); } + /** + * Indicates if mission urls has expired and there an attempt to renovate them + * + * @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false} + */ + public boolean isRecovering() { + return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive(); + } + private boolean doPostprocessing() { if (psAlgorithm == null || psState == 2) return true; @@ -742,10 +750,8 @@ public class DownloadMission extends Mission { } } - // set the current download url to null in case if the recovery - // process is canceled. Next time start() method is called the - // recovery will be executed, saving time - urls[current] = null; + errCode = ERROR_NOTHING; + errObject = null; if (recoveryInfo[current].attempts >= maxRetry) { recoveryInfo[current].attempts = 0; diff --git a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java index 9abd93717..5efbd1153 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java @@ -10,10 +10,12 @@ import org.schabi.newpipe.extractor.stream.SubtitlesStream; import org.schabi.newpipe.extractor.stream.VideoStream; import java.io.IOException; +import java.io.InterruptedIOException; import java.net.HttpURLConnection; import java.nio.channels.ClosedByInterruptException; import java.util.List; +import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING; import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; public class DownloadMissionRecover extends Thread { @@ -21,14 +23,17 @@ public class DownloadMissionRecover extends Thread { static final int mID = -3; private final DownloadMission mMission; - private final MissionRecoveryInfo mRecovery; private final Exception mFromError; + private final boolean notInitialized; + private HttpURLConnection mConn; + private MissionRecoveryInfo mRecovery; + private StreamExtractor mExtractor; DownloadMissionRecover(DownloadMission mission, Exception originError) { mMission = mission; mFromError = originError; - mRecovery = mission.recoveryInfo[mission.current]; + notInitialized = mission.blocks == null && mission.current == 0; } @Override @@ -38,28 +43,78 @@ public class DownloadMissionRecover extends Thread { return; } + /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { + resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); + return; + }*/ + try { - /*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) { - resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length())); - return; - }*/ - StreamingService svr = NewPipe.getServiceByUrl(mMission.source); - - if (svr == null) { - throw new RuntimeException("Unknown source service"); - } - - StreamExtractor extractor = svr.getStreamExtractor(mMission.source); - extractor.fetchPage(); - + mExtractor = svr.getStreamExtractor(mMission.source); + mExtractor.fetchPage(); + } catch (InterruptedIOException | ClosedByInterruptException e) { + return; + } catch (Exception e) { if (!mMission.running || super.isInterrupted()) return; + mMission.notifyError(e); + return; + } + // maybe the following check is redundant + if (!mMission.running || super.isInterrupted()) return; + + if (!notInitialized) { + // set the current download url to null in case if the recovery + // process is canceled. Next time start() method is called the + // recovery will be executed, saving time + mMission.urls[mMission.current] = null; + + mRecovery = mMission.recoveryInfo[mMission.current]; + resolveStream(); + return; + } + + Log.w(TAG, "mission is not fully initialized, this will take a while"); + + try { + for (; mMission.current < mMission.urls.length; mMission.current++) { + mRecovery = mMission.recoveryInfo[mMission.current]; + + if (test()) continue; + if (!mMission.running) return; + + resolveStream(); + if (!mMission.running) return; + + // before continue, check if the current stream was resolved + if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) { + break; + } + } + } finally { + mMission.current = 0; + } + + mMission.writeThisToFile(); + + if (!mMission.running || super.isInterrupted()) return; + + mMission.running = false; + mMission.start(); + } + + private void resolveStream() { + if (mExtractor.getErrorMessage() != null) { + mMission.notifyError(mFromError); + return; + } + + try { String url = null; - switch (mMission.kind) { + switch (mRecovery.kind) { case 'a': - for (AudioStream audio : extractor.getAudioStreams()) { + for (AudioStream audio : mExtractor.getAudioStreams()) { if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { url = audio.getUrl(); break; @@ -69,9 +124,9 @@ public class DownloadMissionRecover extends Thread { case 'v': List videoStreams; if (mRecovery.desired2) - videoStreams = extractor.getVideoOnlyStreams(); + videoStreams = mExtractor.getVideoOnlyStreams(); else - videoStreams = extractor.getVideoStreams(); + videoStreams = mExtractor.getVideoStreams(); for (VideoStream video : videoStreams) { if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { url = video.getUrl(); @@ -80,7 +135,7 @@ public class DownloadMissionRecover extends Thread { } break; case 's': - for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) { + for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) { String tag = subtitles.getLanguageTag(); if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { url = subtitles.getURL(); @@ -114,7 +169,7 @@ public class DownloadMissionRecover extends Thread { ////// Validate the http resource doing a range request ///////////////////// try { - mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length); + mConn = mMission.openConnection(url, true, mMission.length - 10, mMission.length); mConn.setRequestProperty("If-Range", mRecovery.validateCondition); mMission.establishConnection(mID, mConn); @@ -140,22 +195,24 @@ public class DownloadMissionRecover extends Thread { if (!mMission.running || e instanceof ClosedByInterruptException) return; throw e; } finally { - this.interrupt(); + disconnect(); } } private void recover(String url, boolean stale) { Log.i(TAG, - String.format("download recovered name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) + String.format("recover() name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) ); + mMission.urls[mMission.current] = url; + mRecovery.attempts = 0; + if (url == null) { mMission.notifyError(ERROR_RESOURCE_GONE, null); return; } - mMission.urls[mMission.current] = url; - mRecovery.attempts = 0; + if (notInitialized) return; if (stale) { mMission.resetState(false, false, DownloadMission.ERROR_NOTHING); @@ -208,15 +265,40 @@ public class DownloadMissionRecover extends Thread { return range; } + private boolean test() { + if (mMission.urls[mMission.current] == null) return false; + + try { + mConn = mMission.openConnection(mMission.urls[mMission.current], true, -1, -1); + mMission.establishConnection(mID, mConn); + + if (mConn.getResponseCode() == 200) return true; + } catch (Exception e) { + // nothing to do + } finally { + disconnect(); + } + + return false; + } + + private void disconnect() { + try { + try { + mConn.getInputStream().close(); + } finally { + mConn.disconnect(); + } + } catch (Exception e) { + // nothing to do + } finally { + mConn = null; + } + } + @Override public void interrupt() { super.interrupt(); - if (mConn != null) { - try { - mConn.disconnect(); - } catch (Exception e) { - // nothing to do - } - } + if (mConn != null) disconnect(); } } diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java index 1d2a4eee7..b0dc793bc 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnable.java @@ -80,7 +80,7 @@ public class DownloadRunnable extends Thread { } try { - mConn = mMission.openConnection(mId, start, end); + mConn = mMission.openConnection(false, start, end); mMission.establishConnection(mId, mConn); // check if the download can be resumed diff --git a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java index b5937c577..e64322b48 100644 --- a/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java +++ b/app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java @@ -35,7 +35,11 @@ public class DownloadRunnableFallback extends Thread { private void dispose() { try { - if (mIs != null) mIs.close(); + try { + if (mIs != null) mIs.close(); + } finally { + mConn.disconnect(); + } } catch (IOException e) { // nothing to do } @@ -68,7 +72,13 @@ public class DownloadRunnableFallback extends Thread { long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start; int mId = 1; - mConn = mMission.openConnection(mId, rangeStart, -1); + mConn = mMission.openConnection(false, rangeStart, -1); + + if (mRetryCount == 0 && rangeStart == -1) { + // workaround: bypass android connection pool + mConn.setRequestProperty("Range", "bytes=0-"); + } + mMission.establishConnection(mId, mConn); // check if the download can be resumed @@ -96,6 +106,8 @@ public class DownloadRunnableFallback extends Thread { mMission.notifyProgress(len); } + dispose(); + // if thread goes interrupted check if the last part is written. This avoid re-download the whole file done = len == -1; } catch (Exception e) { @@ -107,8 +119,8 @@ public class DownloadRunnableFallback extends Thread { if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { // for youtube streams. The url has expired, recover - mMission.doRecover(e); dispose(); + mMission.doRecover(e); return; } @@ -125,8 +137,6 @@ public class DownloadRunnableFallback extends Thread { return; } - dispose(); - if (done) { mMission.notifyFinished(); } else { diff --git a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java index 553ba6d89..bd1d9bc49 100644 --- a/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java +++ b/app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java @@ -16,25 +16,28 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { private static final long serialVersionUID = 0L; //public static final String DIRECT_SOURCE = "direct-source://"; - public MediaFormat format; + MediaFormat format; String desired; boolean desired2; int desiredBitrate; + byte kind; + String validateCondition = null; transient int attempts = 0; - String validateCondition = null; - public MissionRecoveryInfo(@NonNull Stream stream) { if (stream instanceof AudioStream) { desiredBitrate = ((AudioStream) stream).average_bitrate; desired2 = false; + kind = 'a'; } else if (stream instanceof VideoStream) { desired = ((VideoStream) stream).getResolution(); desired2 = ((VideoStream) stream).isVideoOnly(); + kind = 'v'; } else if (stream instanceof SubtitlesStream) { desired = ((SubtitlesStream) stream).getLanguageTag(); desired2 = ((SubtitlesStream) stream).isAutoGenerated(); + kind = 's'; } else { throw new RuntimeException("Unknown stream kind"); } @@ -43,6 +46,38 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { if (format == null) throw new NullPointerException("Stream format cannot be null"); } + @NonNull + @Override + public String toString() { + String info; + StringBuilder str = new StringBuilder(); + str.append("type="); + switch (kind) { + case 'a': + str.append("audio"); + info = "bitrate=" + desiredBitrate; + break; + case 'v': + str.append("video"); + info = "quality=" + desired + " videoOnly=" + desired2; + break; + case 's': + str.append("subtitles"); + info = "language=" + desired + " autoGenerated=" + desired2; + break; + default: + info = ""; + str.append("other"); + } + + str.append(" format=") + .append(format.getName()) + .append(' ') + .append(info); + + return str.toString(); + } + @Override public int describeContents() { return 0; @@ -54,6 +89,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { parcel.writeString(this.desired); parcel.writeInt(this.desired2 ? 0x01 : 0x00); parcel.writeInt(this.desiredBitrate); + parcel.writeByte(this.kind); parcel.writeString(this.validateCondition); } @@ -62,6 +98,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable { this.desired = parcel.readString(); this.desired2 = parcel.readInt() != 0x00; this.desiredBitrate = parcel.readInt(); + this.kind = parcel.readByte(); this.validateCondition = parcel.readString(); } diff --git a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java index 6c6198750..78fd7ea9d 100644 --- a/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java +++ b/app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java @@ -36,6 +36,7 @@ import android.widget.Toast; import org.schabi.newpipe.BuildConfig; import org.schabi.newpipe.R; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.report.ErrorActivity; import org.schabi.newpipe.report.UserAction; import org.schabi.newpipe.util.NavigationHelper; @@ -44,11 +45,11 @@ import java.io.File; import java.lang.ref.WeakReference; import java.net.URI; import java.util.ArrayList; -import java.util.Collections; import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.FinishedMission; import us.shandian.giga.get.Mission; +import us.shandian.giga.get.MissionRecoveryInfo; import us.shandian.giga.io.StoredFileHelper; import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManagerService; @@ -234,7 +235,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb // hide on error // show if current resource length is not fetched // show if length is unknown - h.progress.setMarquee(!hasError && (!mission.isInitialized() || mission.unknownLength)); + h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength)); float progress; if (mission.unknownLength) { @@ -463,13 +464,13 @@ public class MissionAdapter extends Adapter implements Handler.Callb break; case ERROR_POSTPROCESSING: case ERROR_POSTPROCESSING_HOLD: - showError(mission.errObject, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed); + showError(mission, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed); return; case ERROR_INSUFFICIENT_STORAGE: msg = R.string.error_insufficient_storage; break; case ERROR_UNKNOWN_EXCEPTION: - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, R.string.general_error); + showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error); return; case ERROR_PROGRESS_LOST: msg = R.string.error_progress_lost; @@ -486,7 +487,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb } else if (mission.errObject == null) { msgEx = "(not_decelerated_error_code)"; } else { - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, msg); + showError(mission, UserAction.DOWNLOAD_FAILED, msg); return; } break; @@ -503,7 +504,7 @@ public class MissionAdapter extends Adapter implements Handler.Callb if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) { @StringRes final int mMsg = msg; builder.setPositiveButton(R.string.error_report_title, (dialog, which) -> - showError(mission.errObject, UserAction.DOWNLOAD_FAILED, mMsg) + showError(mission, UserAction.DOWNLOAD_FAILED, mMsg) ); } @@ -513,13 +514,30 @@ public class MissionAdapter extends Adapter implements Handler.Callb .show(); } - private void showError(Exception exception, UserAction action, @StringRes int reason) { + private void showError(DownloadMission mission, UserAction action, @StringRes int reason) { + StringBuilder request = new StringBuilder(256); + request.append(mission.source); + + request.append(" ["); + if (mission.recoveryInfo != null) { + for (MissionRecoveryInfo recovery : mission.recoveryInfo) + request.append(" {").append(recovery.toString()).append("} "); + } + request.append("]"); + + String service; + try { + service = NewPipe.getServiceByUrl(mission.source).getServiceInfo().getName(); + } catch (Exception e) { + service = "-"; + } + ErrorActivity.reportError( mContext, - Collections.singletonList(exception), + mission.errObject, null, null, - ErrorActivity.ErrorInfo.make(action, "-", "-", reason) + ErrorActivity.ErrorInfo.make(action, service, request.toString(), reason) ); } diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 2f69e62cb..b14aab94b 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -453,7 +453,7 @@ No hay suficiente espacio disponible en el dispositivo Se perdió el progreso porque el archivo fue eliminado Tiempo de espera excedido - El recurso solicitado ya no esta disponible + No se puede recuperar esta descarga Preguntar dónde descargar Se preguntará dónde guardar cada descarga Se le preguntará dónde guardar cada descarga. diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 2917fb9fd..f929e0d2b 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -557,7 +557,7 @@ No space left on device Progress lost, because the file was deleted Connection timeout - The solicited resource is not available anymore + Cannot recover this download Clear finished downloads Are you sure? Continue your %s pending transfers from Downloads