baalajimaestro/NewPipe
1
0
Fork 0
Code Issues Pull requests Projects Releases 5 Packages Wiki Activity

misc changes

Browse source 
* OggFromWebMWriter: rewrite (again), reduce iterations over the input. Works as-is (video streams are not supported)
* WebMReader: use int for SimpleBlock.dataSize instead of long
* Download Recovery: allow recovering uninitialized downloads
* check range-requests using HEAD method instead of GET
* DownloadRunnableFallback: add workaround for 32kB/s issue, unknown issue origin, wont fix
* reporting downloads errors now include the source url with the selected quality and format
This commit is contained in:
kapodamy 2019-09-30 23:52:49 -03:00
parent 570738190d
commit 4292ca94ff
11 changed files with 294 additions and 248 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

216
app/src/main/java/org/schabi/newpipe/streams/OggFromWebMWriter.java
View file

@ -12,8 +12,6 @@ import java.io.Closeable;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.ByteOrder; import java.nio.ByteOrder;
import java.util.ArrayList;
import java.util.Random;
import javax.annotation.Nullable; import javax.annotation.Nullable;
@ -23,15 +21,13 @@ import javax.annotation.Nullable;
public class OggFromWebMWriter implements Closeable { public class OggFromWebMWriter implements Closeable {
private static final byte FLAG_UNSET = 0x00; private static final byte FLAG_UNSET = 0x00;
private static final byte FLAG_CONTINUED = 0x01; //private static final byte FLAG_CONTINUED = 0x01;
private static final byte FLAG_FIRST = 0x02; private static final byte FLAG_FIRST = 0x02;
private static final byte FLAG_LAST = 0x04; private static final byte FLAG_LAST = 0x04;
private final static byte HEADER_CHECKSUM_OFFSET = 22; private final static byte HEADER_CHECKSUM_OFFSET = 22;
private final static byte HEADER_SIZE = 27; private final static byte HEADER_SIZE = 27;
private final static short BUFFER_SIZE = 8 * 1024;// 8KiB
private final static int TIME_SCALE_NS = 1000000000; private final static int TIME_SCALE_NS = 1000000000;
private boolean done = false; private boolean done = false;
@ -43,7 +39,6 @@ public class OggFromWebMWriter implements Closeable {
private int sequence_count = 0; private int sequence_count = 0;
private final int STREAM_ID; private final int STREAM_ID;
private byte packet_flag = FLAG_FIRST; private byte packet_flag = FLAG_FIRST;
private int track_index = 0;
private WebMReader webm = null; private WebMReader webm = null;
private WebMTrack webm_track = null; private WebMTrack webm_track = null;
@ -71,7 +66,7 @@ public class OggFromWebMWriter implements Closeable {
this.source = source; this.source = source;
this.output = target; this.output = target;
this.STREAM_ID = (new Random(System.currentTimeMillis())).nextInt(); this.STREAM_ID = (int) System.currentTimeMillis();
populate_crc32_table(); populate_crc32_table();
} }
@ -130,7 +125,6 @@ public class OggFromWebMWriter implements Closeable {
try { try {
webm_track = webm.selectTrack(trackIndex); webm_track = webm.selectTrack(trackIndex);
track_index = trackIndex;
} finally { } finally {
parsed = true; parsed = true;
} }
@ -154,8 +148,11 @@ public class OggFromWebMWriter implements Closeable {
public void build() throws IOException { public void build() throws IOException {
float resolution; float resolution;
int read; SimpleBlock bloq;
byte[] buffer; ByteBuffer header = ByteBuffer.allocate(27 + (255 * 255));
ByteBuffer page = ByteBuffer.allocate(64 * 1024);
header.order(ByteOrder.LITTLE_ENDIAN);
/* step 1: get the amount of frames per seconds */ /* step 1: get the amount of frames per seconds */
switch (webm_track.kind) { switch (webm_track.kind) {
@ -176,57 +173,32 @@ public class OggFromWebMWriter implements Closeable {
throw new RuntimeException("not implemented"); throw new RuntimeException("not implemented");
} }
/* step 2a: create packet with code init data */ /* step 2: create packet with code init data */
ArrayList<byte[]> data_extra = new ArrayList<>(4);
if (webm_track.codecPrivate != null) { if (webm_track.codecPrivate != null) {
addPacketSegment(webm_track.codecPrivate.length); addPacketSegment(webm_track.codecPrivate.length);
ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + webm_track.codecPrivate.length); make_packetHeader(0x00, header, webm_track.codecPrivate);
write(header);
make_packetHeader(0x00, buff, webm_track.codecPrivate); output.write(webm_track.codecPrivate);
data_extra.add(buff.array());
} }
/* step 2b: create packet with metadata */ /* step 3: create packet with metadata */
buffer = make_metadata(); byte[] buffer = make_metadata();
if (buffer != null) { if (buffer != null) {
addPacketSegment(buffer.length); addPacketSegment(buffer.length);
ByteBuffer buff = byte_buffer(HEADER_SIZE + segment_table_size + buffer.length); make_packetHeader(0x00, header, buffer);
write(header);
make_packetHeader(0x00, buff, buffer); output.write(buffer);
data_extra.add(buff.array());
} }
/* step 4: calculate amount of packets */
/* step 3: calculate amount of packets */
SimpleBlock bloq;
int reserve_header = 0;
int headers_amount = 0;
while (webm_segment != null) { while (webm_segment != null) {
bloq = getNextBlock(); bloq = getNextBlock();
if (addPacketSegment(bloq)) { if (bloq != null && addPacketSegment(bloq)) {
continue; int pos = page.position();
} //noinspection ResultOfMethodCallIgnored
bloq.data.read(page.array(), pos, bloq.dataSize);
reserve_header += HEADER_SIZE + segment_table_size;// header size page.position(pos + bloq.dataSize);
clearSegmentTable();
webm_block = bloq;
headers_amount++;
}
/* step 4: create packet headers */
rewind_source();
ByteBuffer headers = byte_buffer(reserve_header);
short[] headers_size = new short[headers_amount];
int header_index = 0;
while (webm_segment != null) {
bloq = getNextBlock();
if (addPacketSegment(bloq)) {
continue; continue;
} }
@ -251,75 +223,21 @@ public class OggFromWebMWriter implements Closeable {
elapsed_ns = elapsed_ns / TIME_SCALE_NS; elapsed_ns = elapsed_ns / TIME_SCALE_NS;
elapsed_ns = Math.ceil(elapsed_ns * resolution); elapsed_ns = Math.ceil(elapsed_ns * resolution);
// create header // create header and calculate page checksum
headers_size[header_index++] = make_packetHeader((long) elapsed_ns, headers, null); int checksum = make_packetHeader((long) elapsed_ns, header, null);
checksum = calc_crc32(checksum, page.array(), page.position());
header.putInt(HEADER_CHECKSUM_OFFSET, checksum);
// dump data
write(header);
write(page);
webm_block = bloq; webm_block = bloq;
} }
/* step 5: calculate checksums */
rewind_source();
int offset = 0;
buffer = new byte[BUFFER_SIZE];
for (header_index = 0; header_index < headers_size.length; header_index++) {
int checksum_offset = offset + HEADER_CHECKSUM_OFFSET;
int checksum = headers.getInt(checksum_offset);
while (webm_segment != null) {
bloq = getNextBlock();
if (!addPacketSegment(bloq)) {
clearSegmentTable();
webm_block = bloq;
break;
}
// calculate page checksum
while ((read = bloq.data.read(buffer)) > 0) {
checksum = calc_crc32(checksum, buffer, 0, read);
}
}
headers.putInt(checksum_offset, checksum);
offset += headers_size[header_index];
}
/* step 6: write extra headers */
rewind_source();
for (byte[] buff : data_extra) {
output.write(buff);
}
/* step 7: write stream packets */
byte[] headers_buffers = headers.array();
offset = 0;
buffer = new byte[BUFFER_SIZE];
for (header_index = 0; header_index < headers_size.length; header_index++) {
output.write(headers_buffers, offset, headers_size[header_index]);
offset += headers_size[header_index];
while (webm_segment != null) {
bloq = getNextBlock();
if (addPacketSegment(bloq)) {
while ((read = bloq.data.read(buffer)) > 0) {
output.write(buffer, 0, read);
}
} else {
clearSegmentTable();
webm_block = bloq;
break;
}
}
}
} }
private short make_packetHeader(long gran_pos, ByteBuffer buffer, byte[] immediate_page) { private int make_packetHeader(long gran_pos, @NonNull ByteBuffer buffer, byte[] immediate_page) {
int offset = buffer.position();
short length = HEADER_SIZE; short length = HEADER_SIZE;
buffer.putInt(0x5367674f);// "OggS" binary string in little-endian buffer.putInt(0x5367674f);// "OggS" binary string in little-endian
@ -340,17 +258,15 @@ public class OggFromWebMWriter implements Closeable {
clearSegmentTable();// clear segment table for next header clearSegmentTable();// clear segment table for next header
int checksum_crc32 = calc_crc32(0x00, buffer.array(), offset, length); int checksum_crc32 = calc_crc32(0x00, buffer.array(), length);
if (immediate_page != null) { if (immediate_page != null) {
checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, 0, immediate_page.length); checksum_crc32 = calc_crc32(checksum_crc32, immediate_page, immediate_page.length);
System.arraycopy(immediate_page, 0, buffer.array(), length, immediate_page.length); buffer.putInt(HEADER_CHECKSUM_OFFSET, checksum_crc32);
segment_table_next_timestamp -= TIME_SCALE_NS; segment_table_next_timestamp -= TIME_SCALE_NS;
} }
buffer.putInt(offset + HEADER_CHECKSUM_OFFSET, checksum_crc32); return checksum_crc32;
return length;
} }
@Nullable @Nullable
@ -358,7 +274,7 @@ public class OggFromWebMWriter implements Closeable {
if ("A_OPUS".equals(webm_track.codecId)) { if ("A_OPUS".equals(webm_track.codecId)) {
return new byte[]{ return new byte[]{
0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string 0x4F, 0x70, 0x75, 0x73, 0x54, 0x61, 0x67, 0x73,// "OpusTags" binary string
0x07, 0x00, 0x00, 0x00,// writing application string size 0x07, 0x00, 0x00, 0x00,// writting application string size
0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string
0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags) 0x00, 0x00, 0x00, 0x00// additional tags count (zero means no tags)
}; };
@ -366,7 +282,7 @@ public class OggFromWebMWriter implements Closeable {
return new byte[]{ return new byte[]{
0x03,// ???????? 0x03,// ????????
0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string 0x76, 0x6f, 0x72, 0x62, 0x69, 0x73,// "vorbis" binary string
0x07, 0x00, 0x00, 0x00,// writing application string size 0x07, 0x00, 0x00, 0x00,// writting application string size
0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string 0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65,// "NewPipe" binary string
0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags) 0x01, 0x00, 0x00, 0x00,// additional tags count (zero means no tags)
@ -387,22 +303,9 @@ public class OggFromWebMWriter implements Closeable {
return null; return null;
} }
private void rewind_source() throws IOException { private void write(ByteBuffer buffer) throws IOException {
source.rewind(); output.write(buffer.array(), 0, buffer.position());
buffer.position(0);
webm = new WebMReader(source);
webm.parse();
webm_track = webm.selectTrack(track_index);
webm_segment = webm.getNextSegment();
webm_cluster = null;
webm_block = null;
webm_block_last_timecode = 0L;
segment_table_next_timestamp = TIME_SCALE_NS;
}
private ByteBuffer byte_buffer(int size) {
return ByteBuffer.allocate(size).order(ByteOrder.LITTLE_ENDIAN);
} }
//<editor-fold defaultstate="collapsed" desc="WebM track handling"> //<editor-fold defaultstate="collapsed" desc="WebM track handling">
@ -460,41 +363,32 @@ public class OggFromWebMWriter implements Closeable {
//<editor-fold defaultstate="collapsed" desc="Segment table writing"> //<editor-fold defaultstate="collapsed" desc="Segment table writing">
private void clearSegmentTable() { private void clearSegmentTable() {
if (packet_flag != FLAG_CONTINUED) { segment_table_next_timestamp += TIME_SCALE_NS;
segment_table_next_timestamp += TIME_SCALE_NS; packet_flag = FLAG_UNSET;
packet_flag = FLAG_UNSET;
}
segment_table_size = 0; segment_table_size = 0;
} }
private boolean addPacketSegment(SimpleBlock block) { private boolean addPacketSegment(SimpleBlock block) {
if (block == null) {
return false;
}
long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay; long timestamp = block.absoluteTimeCodeNs + webm_track.codecDelay;
if (timestamp >= segment_table_next_timestamp) { if (timestamp >= segment_table_next_timestamp) {
return false; return false;
} }
boolean result = addPacketSegment((int) block.dataSize); return addPacketSegment(block.dataSize);
if (!result && segment_table_next_timestamp < timestamp) {
// WARNING: ¡¡¡¡ not implemented (lack of documentation) !!!!
packet_flag = FLAG_CONTINUED;
}
return result;
} }
private boolean addPacketSegment(int size) { private boolean addPacketSegment(int size) {
if (size > 65025) {
throw new UnsupportedOperationException("page size cannot be larger than 65025");
}
int available = (segment_table.length - segment_table_size) * 255; int available = (segment_table.length - segment_table_size) * 255;
boolean extra = size == 255; boolean extra = (size % 255) == 0;
if (extra) { if (extra) {
// add a zero byte entry in the table // add a zero byte entry in the table
// required to indicate the sample size is exactly 255 // required to indicate the sample size is multiple of 255
available -= 255; available -= 255;
} }
@ -528,12 +422,10 @@ public class OggFromWebMWriter implements Closeable {
} }
} }
private int calc_crc32(int initial_crc, byte[] buffer, int offset, int size) { private int calc_crc32(int initial_crc, byte[] buffer, int size) {
size += offset; for (int i = 0; i < size; i++) {
for (; offset < size; offset++) {
int reg = (initial_crc >>> 24) & 0xff; int reg = (initial_crc >>> 24) & 0xff;
initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[offset] & 0xff)]; initial_crc = (initial_crc << 8) ^ crc32_table[reg ^ (buffer[i] & 0xff)];
} }
return initial_crc; return initial_crc;

4
app/src/main/java/org/schabi/newpipe/streams/WebMReader.java
View file

@ -368,7 +368,7 @@ public class WebMReader {
obj.trackNumber = readEncodedNumber(); obj.trackNumber = readEncodedNumber();
obj.relativeTimeCode = stream.readShort(); obj.relativeTimeCode = stream.readShort();
obj.flags = (byte) stream.read(); obj.flags = (byte) stream.read();
obj.dataSize = (ref.offset + ref.size) - stream.position(); obj.dataSize = (int) ((ref.offset + ref.size) - stream.position());
obj.createdFromBlock = ref.type == ID_Block; obj.createdFromBlock = ref.type == ID_Block;
// NOTE: lacing is not implemented, and will be mixed with the stream data // NOTE: lacing is not implemented, and will be mixed with the stream data
@ -465,7 +465,7 @@ public class WebMReader {
public short relativeTimeCode; public short relativeTimeCode;
public long absoluteTimeCodeNs; public long absoluteTimeCodeNs;
public byte flags; public byte flags;
public long dataSize; public int dataSize;
private final Element ref; private final Element ref;
public boolean isKeyframe() { public boolean isKeyframe() {

35
app/src/main/java/us/shandian/giga/get/DownloadInitializer.java
View file

@ -14,6 +14,7 @@ import java.nio.channels.ClosedByInterruptException;
import us.shandian.giga.util.Utility; import us.shandian.giga.util.Utility;
import static org.schabi.newpipe.BuildConfig.DEBUG; import static org.schabi.newpipe.BuildConfig.DEBUG;
import static us.shandian.giga.get.DownloadMission.ERROR_HTTP_FORBIDDEN;
public class DownloadInitializer extends Thread { public class DownloadInitializer extends Thread {
private final static String TAG = "DownloadInitializer"; private final static String TAG = "DownloadInitializer";
@ -29,9 +30,9 @@ public class DownloadInitializer extends Thread {
mConn = null; mConn = null;
} }
private static void safeClose(HttpURLConnection con) { private void dispose() {
try { try {
con.getInputStream().close(); mConn.getInputStream().close();
} catch (Exception e) { } catch (Exception e) {
// nothing to do // nothing to do
} }
@ -52,9 +53,9 @@ public class DownloadInitializer extends Thread {
long lowestSize = Long.MAX_VALUE; long lowestSize = Long.MAX_VALUE;
for (int i = 0; i < mMission.urls.length && mMission.running; i++) { for (int i = 0; i < mMission.urls.length && mMission.running; i++) {
mConn = mMission.openConnection(mMission.urls[i], mId, -1, -1); mConn = mMission.openConnection(mMission.urls[i], true, -1, -1);
mMission.establishConnection(mId, mConn); mMission.establishConnection(mId, mConn);
safeClose(mConn); dispose();
if (Thread.interrupted()) return; if (Thread.interrupted()) return;
long length = Utility.getContentLength(mConn); long length = Utility.getContentLength(mConn);
@ -82,9 +83,9 @@ public class DownloadInitializer extends Thread {
} }
} else { } else {
// ask for the current resource length // ask for the current resource length
mConn = mMission.openConnection(mId, -1, -1); mConn = mMission.openConnection(true, -1, -1);
mMission.establishConnection(mId, mConn); mMission.establishConnection(mId, mConn);
safeClose(mConn); dispose();
if (!mMission.running || Thread.interrupted()) return; if (!mMission.running || Thread.interrupted()) return;
@ -108,9 +109,9 @@ public class DownloadInitializer extends Thread {
} }
} else { } else {
// Open again // Open again
mConn = mMission.openConnection(mId, mMission.length - 10, mMission.length); mConn = mMission.openConnection(true, mMission.length - 10, mMission.length);
mMission.establishConnection(mId, mConn); mMission.establishConnection(mId, mConn);
safeClose(mConn); dispose();
if (!mMission.running || Thread.interrupted()) return; if (!mMission.running || Thread.interrupted()) return;
@ -171,7 +172,14 @@ public class DownloadInitializer extends Thread {
} catch (InterruptedIOException | ClosedByInterruptException e) { } catch (InterruptedIOException | ClosedByInterruptException e) {
return; return;
} catch (Exception e) { } catch (Exception e) {
if (!mMission.running) return; if (!mMission.running || super.isInterrupted()) return;
if (e instanceof DownloadMission.HttpError && ((DownloadMission.HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) {
// for youtube streams. The url has expired
interrupt();
mMission.doRecover(e);
return;
}
if (e instanceof IOException && e.getMessage().contains("Permission denied")) { if (e instanceof IOException && e.getMessage().contains("Permission denied")) {
mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e); mMission.notifyError(DownloadMission.ERROR_PERMISSION_DENIED, e);
@ -194,13 +202,6 @@ public class DownloadInitializer extends Thread {
@Override @Override
public void interrupt() { public void interrupt() {
super.interrupt(); super.interrupt();
if (mConn != null) dispose();
if (mConn != null) {
try {
mConn.disconnect();
} catch (Exception e) {
// nothing to do
}
}
} }
} }

36
app/src/main/java/us/shandian/giga/get/DownloadMission.java
View file

@ -204,22 +204,24 @@ public class DownloadMission extends Mission {
/** /**
* Opens a connection * Opens a connection
* *
* @param threadId id of the calling thread, used only for debugging * @param headRequest {@code true} for use {@code HEAD} request method, otherwise, {@code GET} is used
* @param rangeStart range start * @param rangeStart range start
* @param rangeEnd range end * @param rangeEnd range end
* @return a {@link java.net.URLConnection URLConnection} linking to the URL. * @return a {@link java.net.URLConnection URLConnection} linking to the URL.
* @throws IOException if an I/O exception occurs. * @throws IOException if an I/O exception occurs.
*/ */
HttpURLConnection openConnection(int threadId, long rangeStart, long rangeEnd) throws IOException { HttpURLConnection openConnection(boolean headRequest, long rangeStart, long rangeEnd) throws IOException {
return openConnection(urls[current], threadId, rangeStart, rangeEnd); return openConnection(urls[current], headRequest, rangeStart, rangeEnd);
} }
HttpURLConnection openConnection(String url, int threadId, long rangeStart, long rangeEnd) throws IOException { HttpURLConnection openConnection(String url, boolean headRequest, long rangeStart, long rangeEnd) throws IOException {
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection(); HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
conn.setInstanceFollowRedirects(true); conn.setInstanceFollowRedirects(true);
conn.setRequestProperty("User-Agent", DownloaderImpl.USER_AGENT); conn.setRequestProperty("User-Agent", DownloaderImpl.USER_AGENT);
conn.setRequestProperty("Accept", "*/*"); conn.setRequestProperty("Accept", "*/*");
if (headRequest) conn.setRequestMethod("HEAD");
// BUG workaround: switching between networks can freeze the download forever // BUG workaround: switching between networks can freeze the download forever
conn.setConnectTimeout(30000); conn.setConnectTimeout(30000);
conn.setReadTimeout(10000); conn.setReadTimeout(10000);
@ -229,10 +231,6 @@ public class DownloadMission extends Mission {
if (rangeEnd > 0) req += rangeEnd; if (rangeEnd > 0) req += rangeEnd;
conn.setRequestProperty("Range", req); conn.setRequestProperty("Range", req);
if (DEBUG) {
Log.d(TAG, threadId + ":" + conn.getRequestProperty("Range"));
}
} }
return conn; return conn;
@ -245,13 +243,14 @@ public class DownloadMission extends Mission {
* @throws HttpError if the HTTP Status-Code is not satisfiable * @throws HttpError if the HTTP Status-Code is not satisfiable
*/ */
void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError { void establishConnection(int threadId, HttpURLConnection conn) throws IOException, HttpError {
conn.connect();
int statusCode = conn.getResponseCode(); int statusCode = conn.getResponseCode();
if (DEBUG) { if (DEBUG) {
Log.d(TAG, threadId + ":Range=" + conn.getRequestProperty("Range"));
Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode); Log.d(TAG, threadId + ":Content-Length=" + conn.getContentLength() + " Code:" + statusCode);
} }
switch (statusCode) { switch (statusCode) {
case 204: case 204:
case 205: case 205:
@ -676,6 +675,15 @@ public class DownloadMission extends Mission {
return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished(); return (isPsFailed() || errCode == ERROR_POSTPROCESSING_HOLD) || isFinished();
} }
/**
* Indicates if mission urls has expired and there an attempt to renovate them
*
* @return {@code true} if the mission is running a recovery procedure, otherwise, {@code false}
*/
public boolean isRecovering() {
return threads != null && threads.length > 0 && threads[0] instanceof DownloadRunnable && threads[0].isAlive();
}
private boolean doPostprocessing() { private boolean doPostprocessing() {
if (psAlgorithm == null || psState == 2) return true; if (psAlgorithm == null || psState == 2) return true;
@ -742,10 +750,8 @@ public class DownloadMission extends Mission {
} }
} }
// set the current download url to null in case if the recovery errCode = ERROR_NOTHING;
// process is canceled. Next time start() method is called the errObject = null;
// recovery will be executed, saving time
urls[current] = null;
if (recoveryInfo[current].attempts >= maxRetry) { if (recoveryInfo[current].attempts >= maxRetry) {
recoveryInfo[current].attempts = 0; recoveryInfo[current].attempts = 0;

146
app/src/main/java/us/shandian/giga/get/DownloadMissionRecover.java
View file

@ -10,10 +10,12 @@ import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream; import org.schabi.newpipe.extractor.stream.VideoStream;
import java.io.IOException; import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.HttpURLConnection; import java.net.HttpURLConnection;
import java.nio.channels.ClosedByInterruptException; import java.nio.channels.ClosedByInterruptException;
import java.util.List; import java.util.List;
import static us.shandian.giga.get.DownloadMission.ERROR_NOTHING;
import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE; import static us.shandian.giga.get.DownloadMission.ERROR_RESOURCE_GONE;
public class DownloadMissionRecover extends Thread { public class DownloadMissionRecover extends Thread {
@ -21,14 +23,17 @@ public class DownloadMissionRecover extends Thread {
static final int mID = -3; static final int mID = -3;
private final DownloadMission mMission; private final DownloadMission mMission;
private final MissionRecoveryInfo mRecovery;
private final Exception mFromError; private final Exception mFromError;
private final boolean notInitialized;
private HttpURLConnection mConn; private HttpURLConnection mConn;
private MissionRecoveryInfo mRecovery;
private StreamExtractor mExtractor;
DownloadMissionRecover(DownloadMission mission, Exception originError) { DownloadMissionRecover(DownloadMission mission, Exception originError) {
mMission = mission; mMission = mission;
mFromError = originError; mFromError = originError;
mRecovery = mission.recoveryInfo[mission.current]; notInitialized = mission.blocks == null && mission.current == 0;
} }
@Override @Override
@ -38,28 +43,78 @@ public class DownloadMissionRecover extends Thread {
return; return;
} }
/*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) {
resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length()));
return;
}*/
try { try {
/*if (mMission.source.startsWith(MissionRecoveryInfo.DIRECT_SOURCE)) {
resolve(mMission.source.substring(MissionRecoveryInfo.DIRECT_SOURCE.length()));
return;
}*/
StreamingService svr = NewPipe.getServiceByUrl(mMission.source); StreamingService svr = NewPipe.getServiceByUrl(mMission.source);
mExtractor = svr.getStreamExtractor(mMission.source);
if (svr == null) { mExtractor.fetchPage();
throw new RuntimeException("Unknown source service"); } catch (InterruptedIOException | ClosedByInterruptException e) {
} return;
} catch (Exception e) {
StreamExtractor extractor = svr.getStreamExtractor(mMission.source);
extractor.fetchPage();
if (!mMission.running || super.isInterrupted()) return; if (!mMission.running || super.isInterrupted()) return;
mMission.notifyError(e);
return;
}
// maybe the following check is redundant
if (!mMission.running || super.isInterrupted()) return;
if (!notInitialized) {
// set the current download url to null in case if the recovery
// process is canceled. Next time start() method is called the
// recovery will be executed, saving time
mMission.urls[mMission.current] = null;
mRecovery = mMission.recoveryInfo[mMission.current];
resolveStream();
return;
}
Log.w(TAG, "mission is not fully initialized, this will take a while");
try {
for (; mMission.current < mMission.urls.length; mMission.current++) {
mRecovery = mMission.recoveryInfo[mMission.current];
if (test()) continue;
if (!mMission.running) return;
resolveStream();
if (!mMission.running) return;
// before continue, check if the current stream was resolved
if (mMission.urls[mMission.current] == null || mMission.errCode != ERROR_NOTHING) {
break;
}
}
} finally {
mMission.current = 0;
}
mMission.writeThisToFile();
if (!mMission.running || super.isInterrupted()) return;
mMission.running = false;
mMission.start();
}
private void resolveStream() {
if (mExtractor.getErrorMessage() != null) {
mMission.notifyError(mFromError);
return;
}
try {
String url = null; String url = null;
switch (mMission.kind) { switch (mRecovery.kind) {
case 'a': case 'a':
for (AudioStream audio : extractor.getAudioStreams()) { for (AudioStream audio : mExtractor.getAudioStreams()) {
if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) { if (audio.average_bitrate == mRecovery.desiredBitrate && audio.getFormat() == mRecovery.format) {
url = audio.getUrl(); url = audio.getUrl();
break; break;
@ -69,9 +124,9 @@ public class DownloadMissionRecover extends Thread {
case 'v': case 'v':
List<VideoStream> videoStreams; List<VideoStream> videoStreams;
if (mRecovery.desired2) if (mRecovery.desired2)
videoStreams = extractor.getVideoOnlyStreams(); videoStreams = mExtractor.getVideoOnlyStreams();
else else
videoStreams = extractor.getVideoStreams(); videoStreams = mExtractor.getVideoStreams();
for (VideoStream video : videoStreams) { for (VideoStream video : videoStreams) {
if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) { if (video.resolution.equals(mRecovery.desired) && video.getFormat() == mRecovery.format) {
url = video.getUrl(); url = video.getUrl();
@ -80,7 +135,7 @@ public class DownloadMissionRecover extends Thread {
} }
break; break;
case 's': case 's':
for (SubtitlesStream subtitles : extractor.getSubtitles(mRecovery.format)) { for (SubtitlesStream subtitles : mExtractor.getSubtitles(mRecovery.format)) {
String tag = subtitles.getLanguageTag(); String tag = subtitles.getLanguageTag();
if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) { if (tag.equals(mRecovery.desired) && subtitles.isAutoGenerated() == mRecovery.desired2) {
url = subtitles.getURL(); url = subtitles.getURL();
@ -114,7 +169,7 @@ public class DownloadMissionRecover extends Thread {
////// Validate the http resource doing a range request ////// Validate the http resource doing a range request
///////////////////// /////////////////////
try { try {
mConn = mMission.openConnection(url, mID, mMission.length - 10, mMission.length); mConn = mMission.openConnection(url, true, mMission.length - 10, mMission.length);
mConn.setRequestProperty("If-Range", mRecovery.validateCondition); mConn.setRequestProperty("If-Range", mRecovery.validateCondition);
mMission.establishConnection(mID, mConn); mMission.establishConnection(mID, mConn);
@ -140,22 +195,24 @@ public class DownloadMissionRecover extends Thread {
if (!mMission.running || e instanceof ClosedByInterruptException) return; if (!mMission.running || e instanceof ClosedByInterruptException) return;
throw e; throw e;
} finally { } finally {
this.interrupt(); disconnect();
} }
} }
private void recover(String url, boolean stale) { private void recover(String url, boolean stale) {
Log.i(TAG, Log.i(TAG,
String.format("download recovered name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url) String.format("recover() name=%s isStale=%s url=%s", mMission.storage.getName(), stale, url)
); );
mMission.urls[mMission.current] = url;
mRecovery.attempts = 0;
if (url == null) { if (url == null) {
mMission.notifyError(ERROR_RESOURCE_GONE, null); mMission.notifyError(ERROR_RESOURCE_GONE, null);
return; return;
} }
mMission.urls[mMission.current] = url; if (notInitialized) return;
mRecovery.attempts = 0;
if (stale) { if (stale) {
mMission.resetState(false, false, DownloadMission.ERROR_NOTHING); mMission.resetState(false, false, DownloadMission.ERROR_NOTHING);
@ -208,15 +265,40 @@ public class DownloadMissionRecover extends Thread {
return range; return range;
} }
private boolean test() {
if (mMission.urls[mMission.current] == null) return false;
try {
mConn = mMission.openConnection(mMission.urls[mMission.current], true, -1, -1);
mMission.establishConnection(mID, mConn);
if (mConn.getResponseCode() == 200) return true;
} catch (Exception e) {
// nothing to do
} finally {
disconnect();
}
return false;
}
private void disconnect() {
try {
try {
mConn.getInputStream().close();
} finally {
mConn.disconnect();
}
} catch (Exception e) {
// nothing to do
} finally {
mConn = null;
}
}
@Override @Override
public void interrupt() { public void interrupt() {
super.interrupt(); super.interrupt();
if (mConn != null) { if (mConn != null) disconnect();
try {
mConn.disconnect();
} catch (Exception e) {
// nothing to do
}
}
} }
} }

2
app/src/main/java/us/shandian/giga/get/DownloadRunnable.java
View file

@ -80,7 +80,7 @@ public class DownloadRunnable extends Thread {
} }
try { try {
mConn = mMission.openConnection(mId, start, end); mConn = mMission.openConnection(false, start, end);
mMission.establishConnection(mId, mConn); mMission.establishConnection(mId, mConn);
// check if the download can be resumed // check if the download can be resumed

20
app/src/main/java/us/shandian/giga/get/DownloadRunnableFallback.java
View file

@ -35,7 +35,11 @@ public class DownloadRunnableFallback extends Thread {
private void dispose() { private void dispose() {
try { try {
if (mIs != null) mIs.close(); try {
if (mIs != null) mIs.close();
} finally {
mConn.disconnect();
}
} catch (IOException e) { } catch (IOException e) {
// nothing to do // nothing to do
} }
@ -68,7 +72,13 @@ public class DownloadRunnableFallback extends Thread {
long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start; long rangeStart = (mMission.unknownLength || start < 1) ? -1 : start;
int mId = 1; int mId = 1;
mConn = mMission.openConnection(mId, rangeStart, -1); mConn = mMission.openConnection(false, rangeStart, -1);
if (mRetryCount == 0 && rangeStart == -1) {
// workaround: bypass android connection pool
mConn.setRequestProperty("Range", "bytes=0-");
}
mMission.establishConnection(mId, mConn); mMission.establishConnection(mId, mConn);
// check if the download can be resumed // check if the download can be resumed
@ -96,6 +106,8 @@ public class DownloadRunnableFallback extends Thread {
mMission.notifyProgress(len); mMission.notifyProgress(len);
} }
dispose();
// if thread goes interrupted check if the last part is written. This avoid re-download the whole file // if thread goes interrupted check if the last part is written. This avoid re-download the whole file
done = len == -1; done = len == -1;
} catch (Exception e) { } catch (Exception e) {
@ -107,8 +119,8 @@ public class DownloadRunnableFallback extends Thread {
if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) { if (e instanceof HttpError && ((HttpError) e).statusCode == ERROR_HTTP_FORBIDDEN) {
// for youtube streams. The url has expired, recover // for youtube streams. The url has expired, recover
mMission.doRecover(e);
dispose(); dispose();
mMission.doRecover(e);
return; return;
} }
@ -125,8 +137,6 @@ public class DownloadRunnableFallback extends Thread {
return; return;
} }
dispose();
if (done) { if (done) {
mMission.notifyFinished(); mMission.notifyFinished();
} else { } else {

43
app/src/main/java/us/shandian/giga/get/MissionRecoveryInfo.java
View file

@ -16,25 +16,28 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
private static final long serialVersionUID = 0L; private static final long serialVersionUID = 0L;
//public static final String DIRECT_SOURCE = "direct-source://"; //public static final String DIRECT_SOURCE = "direct-source://";
public MediaFormat format; MediaFormat format;
String desired; String desired;
boolean desired2; boolean desired2;
int desiredBitrate; int desiredBitrate;
byte kind;
String validateCondition = null;
transient int attempts = 0; transient int attempts = 0;
String validateCondition = null;
public MissionRecoveryInfo(@NonNull Stream stream) { public MissionRecoveryInfo(@NonNull Stream stream) {
if (stream instanceof AudioStream) { if (stream instanceof AudioStream) {
desiredBitrate = ((AudioStream) stream).average_bitrate; desiredBitrate = ((AudioStream) stream).average_bitrate;
desired2 = false; desired2 = false;
kind = 'a';
} else if (stream instanceof VideoStream) { } else if (stream instanceof VideoStream) {
desired = ((VideoStream) stream).getResolution(); desired = ((VideoStream) stream).getResolution();
desired2 = ((VideoStream) stream).isVideoOnly(); desired2 = ((VideoStream) stream).isVideoOnly();
kind = 'v';
} else if (stream instanceof SubtitlesStream) { } else if (stream instanceof SubtitlesStream) {
desired = ((SubtitlesStream) stream).getLanguageTag(); desired = ((SubtitlesStream) stream).getLanguageTag();
desired2 = ((SubtitlesStream) stream).isAutoGenerated(); desired2 = ((SubtitlesStream) stream).isAutoGenerated();
kind = 's';
} else { } else {
throw new RuntimeException("Unknown stream kind"); throw new RuntimeException("Unknown stream kind");
} }
@ -43,6 +46,38 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
if (format == null) throw new NullPointerException("Stream format cannot be null"); if (format == null) throw new NullPointerException("Stream format cannot be null");
} }
@NonNull
@Override
public String toString() {
String info;
StringBuilder str = new StringBuilder();
str.append("type=");
switch (kind) {
case 'a':
str.append("audio");
info = "bitrate=" + desiredBitrate;
break;
case 'v':
str.append("video");
info = "quality=" + desired + " videoOnly=" + desired2;
break;
case 's':
str.append("subtitles");
info = "language=" + desired + " autoGenerated=" + desired2;
break;
default:
info = "";
str.append("other");
}
str.append(" format=")
.append(format.getName())
.append(' ')
.append(info);
return str.toString();
}
@Override @Override
public int describeContents() { public int describeContents() {
return 0; return 0;
@ -54,6 +89,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
parcel.writeString(this.desired); parcel.writeString(this.desired);
parcel.writeInt(this.desired2 ? 0x01 : 0x00); parcel.writeInt(this.desired2 ? 0x01 : 0x00);
parcel.writeInt(this.desiredBitrate); parcel.writeInt(this.desiredBitrate);
parcel.writeByte(this.kind);
parcel.writeString(this.validateCondition); parcel.writeString(this.validateCondition);
} }
@ -62,6 +98,7 @@ public class MissionRecoveryInfo implements Serializable, Parcelable {
this.desired = parcel.readString(); this.desired = parcel.readString();
this.desired2 = parcel.readInt() != 0x00; this.desired2 = parcel.readInt() != 0x00;
this.desiredBitrate = parcel.readInt(); this.desiredBitrate = parcel.readInt();
this.kind = parcel.readByte();
this.validateCondition = parcel.readString(); this.validateCondition = parcel.readString();
} }

36
app/src/main/java/us/shandian/giga/ui/adapter/MissionAdapter.java
View file

@ -36,6 +36,7 @@ import android.widget.Toast;
import org.schabi.newpipe.BuildConfig; import org.schabi.newpipe.BuildConfig;
import org.schabi.newpipe.R; import org.schabi.newpipe.R;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.report.ErrorActivity; import org.schabi.newpipe.report.ErrorActivity;
import org.schabi.newpipe.report.UserAction; import org.schabi.newpipe.report.UserAction;
import org.schabi.newpipe.util.NavigationHelper; import org.schabi.newpipe.util.NavigationHelper;
@ -44,11 +45,11 @@ import java.io.File;
import java.lang.ref.WeakReference; import java.lang.ref.WeakReference;
import java.net.URI; import java.net.URI;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.get.FinishedMission; import us.shandian.giga.get.FinishedMission;
import us.shandian.giga.get.Mission; import us.shandian.giga.get.Mission;
import us.shandian.giga.get.MissionRecoveryInfo;
import us.shandian.giga.io.StoredFileHelper; import us.shandian.giga.io.StoredFileHelper;
import us.shandian.giga.service.DownloadManager; import us.shandian.giga.service.DownloadManager;
import us.shandian.giga.service.DownloadManagerService; import us.shandian.giga.service.DownloadManagerService;
@ -234,7 +235,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
// hide on error // hide on error
// show if current resource length is not fetched // show if current resource length is not fetched
// show if length is unknown // show if length is unknown
h.progress.setMarquee(!hasError && (!mission.isInitialized() || mission.unknownLength)); h.progress.setMarquee(mission.isRecovering() || !hasError && (!mission.isInitialized() || mission.unknownLength));
float progress; float progress;
if (mission.unknownLength) { if (mission.unknownLength) {
@ -463,13 +464,13 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
break; break;
case ERROR_POSTPROCESSING: case ERROR_POSTPROCESSING:
case ERROR_POSTPROCESSING_HOLD: case ERROR_POSTPROCESSING_HOLD:
showError(mission.errObject, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed); showError(mission, UserAction.DOWNLOAD_POSTPROCESSING, R.string.error_postprocessing_failed);
return; return;
case ERROR_INSUFFICIENT_STORAGE: case ERROR_INSUFFICIENT_STORAGE:
msg = R.string.error_insufficient_storage; msg = R.string.error_insufficient_storage;
break; break;
case ERROR_UNKNOWN_EXCEPTION: case ERROR_UNKNOWN_EXCEPTION:
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, R.string.general_error); showError(mission, UserAction.DOWNLOAD_FAILED, R.string.general_error);
return; return;
case ERROR_PROGRESS_LOST: case ERROR_PROGRESS_LOST:
msg = R.string.error_progress_lost; msg = R.string.error_progress_lost;
@ -486,7 +487,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
} else if (mission.errObject == null) { } else if (mission.errObject == null) {
msgEx = "(not_decelerated_error_code)"; msgEx = "(not_decelerated_error_code)";
} else { } else {
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, msg); showError(mission, UserAction.DOWNLOAD_FAILED, msg);
return; return;
} }
break; break;
@ -503,7 +504,7 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) { if (mission.errObject != null && (mission.errCode < 100 || mission.errCode >= 600)) {
@StringRes final int mMsg = msg; @StringRes final int mMsg = msg;
builder.setPositiveButton(R.string.error_report_title, (dialog, which) -> builder.setPositiveButton(R.string.error_report_title, (dialog, which) ->
showError(mission.errObject, UserAction.DOWNLOAD_FAILED, mMsg) showError(mission, UserAction.DOWNLOAD_FAILED, mMsg)
); );
} }
@ -513,13 +514,30 @@ public class MissionAdapter extends Adapter<ViewHolder> implements Handler.Callb
.show(); .show();
} }
private void showError(Exception exception, UserAction action, @StringRes int reason) { private void showError(DownloadMission mission, UserAction action, @StringRes int reason) {
StringBuilder request = new StringBuilder(256);
request.append(mission.source);
request.append(" [");
if (mission.recoveryInfo != null) {
for (MissionRecoveryInfo recovery : mission.recoveryInfo)
request.append(" {").append(recovery.toString()).append("} ");
}
request.append("]");
String service;
try {
service = NewPipe.getServiceByUrl(mission.source).getServiceInfo().getName();
} catch (Exception e) {
service = "-";
}
ErrorActivity.reportError( ErrorActivity.reportError(
mContext, mContext,
Collections.singletonList(exception), mission.errObject,
null, null,
null, null,
ErrorActivity.ErrorInfo.make(action, "-", "-", reason) ErrorActivity.ErrorInfo.make(action, service, request.toString(), reason)
); );
} }

2
app/src/main/res/values-es/strings.xml
View file

@ -453,7 +453,7 @@
<string name="error_insufficient_storage">No hay suficiente espacio disponible en el dispositivo</string> <string name="error_insufficient_storage">No hay suficiente espacio disponible en el dispositivo</string>
<string name="error_progress_lost">Se perdió el progreso porque el archivo fue eliminado</string> <string name="error_progress_lost">Se perdió el progreso porque el archivo fue eliminado</string>
<string name="error_timeout">Tiempo de espera excedido</string> <string name="error_timeout">Tiempo de espera excedido</string>
<string name="error_download_resource_gone">El recurso solicitado ya no esta disponible</string> <string name="error_download_resource_gone">No se puede recuperar esta descarga</string>
<string name="downloads_storage_ask_title">Preguntar dónde descargar</string> <string name="downloads_storage_ask_title">Preguntar dónde descargar</string>
<string name="downloads_storage_ask_summary">Se preguntará dónde guardar cada descarga</string> <string name="downloads_storage_ask_summary">Se preguntará dónde guardar cada descarga</string>
<string name="downloads_storage_ask_summary_kitkat">Se le preguntará dónde guardar cada descarga. <string name="downloads_storage_ask_summary_kitkat">Se le preguntará dónde guardar cada descarga.

2
app/src/main/res/values/strings.xml
View file

@ -557,7 +557,7 @@
<string name="error_insufficient_storage">No space left on device</string> <string name="error_insufficient_storage">No space left on device</string>
<string name="error_progress_lost">Progress lost, because the file was deleted</string> <string name="error_progress_lost">Progress lost, because the file was deleted</string>
<string name="error_timeout">Connection timeout</string> <string name="error_timeout">Connection timeout</string>
<string name="error_download_resource_gone">The solicited resource is not available anymore</string> <string name="error_download_resource_gone">Cannot recover this download</string>
<string name="clear_finished_download">Clear finished downloads</string> <string name="clear_finished_download">Clear finished downloads</string>
<string name="confirm_prompt">Are you sure?</string> <string name="confirm_prompt">Are you sure?</string>
<string name="msg_pending_downloads">Continue your %s pending transfers from Downloads</string> <string name="msg_pending_downloads">Continue your %s pending transfers from Downloads</string>