baalajimaestro/NewPipe
1
0
Fork 0
Code Issues Pull requests Projects Releases 7 Packages Wiki Activity

misc utils

Browse source 
Also this include:
* Mp4 DASH reader/writter
* WebM reader/writter
* a subtitle converter for Timed Text Markup Language v1 and TranScript (v1, v2 and v3)
* SharpStream to wrap IntputStream and OutputStream in one interface
* custom implementation of DataInputStream
This commit is contained in:
kapodamy 2018-11-15 20:17:22 -03:00 committed by GitHub
parent eb1f56488f
commit f42d077f30
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 3308 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
app/src/main/java/org/schabi/newpipe/streams/DataReader.java Normal file
View file

@ -0,0 +1,103 @@
package org.schabi.newpipe.streams;
import java.io.EOFException;
import java.io.IOException;
import org.schabi.newpipe.streams.io.SharpStream;
/**
* @author kapodamy
*/
public class DataReader {
public final static int SHORT_SIZE = 2;
public final static int LONG_SIZE = 8;
public final static int INTEGER_SIZE = 4;
public final static int FLOAT_SIZE = 4;
private long pos;
public final SharpStream stream;
private final boolean rewind;
public DataReader(SharpStream stream) {
this.rewind = stream.canRewind();
this.stream = stream;
this.pos = 0L;
}
public long position() {
return pos;
}
public final int readInt() throws IOException {
primitiveRead(INTEGER_SIZE);
return primitive[0] << 24 | primitive[1] << 16 | primitive[2] << 8 | primitive[3];
}
public final int read() throws IOException {
int value = stream.read();
if (value == -1) {
throw new EOFException();
}
pos++;
return value;
}
public final long skipBytes(long amount) throws IOException {
amount = stream.skip(amount);
pos += amount;
return amount;
}
public final long readLong() throws IOException {
primitiveRead(LONG_SIZE);
long high = primitive[0] << 24 | primitive[1] << 16 | primitive[2] << 8 | primitive[3];
long low = primitive[4] << 24 | primitive[5] << 16 | primitive[6] << 8 | primitive[7];
return high << 32 | low;
}
public final short readShort() throws IOException {
primitiveRead(SHORT_SIZE);
return (short) (primitive[0] << 8 | primitive[1]);
}
public final int read(byte[] buffer) throws IOException {
return read(buffer, 0, buffer.length);
}
public final int read(byte[] buffer, int offset, int count) throws IOException {
int res = stream.read(buffer, offset, count);
pos += res;
return res;
}
public final boolean available() {
return stream.available() > 0;
}
public void rewind() throws IOException {
stream.rewind();
pos = 0;
}
public boolean canRewind() {
return rewind;
}
private short[] primitive = new short[LONG_SIZE];
private void primitiveRead(int amount) throws IOException {
byte[] buffer = new byte[amount];
int read = stream.read(buffer, 0, amount);
pos += read;
if (read != amount) {
throw new EOFException("Truncated data, missing " + String.valueOf(amount - read) + " bytes");
}
for (int i = 0; i < buffer.length; i++) {
primitive[i] = (short) (buffer[i] & 0xFF);// the "byte" datatype is signed and is very annoying
}
}
}

817
app/src/main/java/org/schabi/newpipe/streams/Mp4DashReader.java Normal file
View file

@ -0,0 +1,817 @@
package org.schabi.newpipe.streams;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.NoSuchElementException;
import org.schabi.newpipe.streams.io.SharpStream;
/**
* @author kapodamy
*/
public class Mp4DashReader {
// <editor-fold defaultState="collapsed" desc="Constants">
private static final int ATOM_MOOF = 0x6D6F6F66;
private static final int ATOM_MFHD = 0x6D666864;
private static final int ATOM_TRAF = 0x74726166;
private static final int ATOM_TFHD = 0x74666864;
private static final int ATOM_TFDT = 0x74666474;
private static final int ATOM_TRUN = 0x7472756E;
private static final int ATOM_MDIA = 0x6D646961;
private static final int ATOM_FTYP = 0x66747970;
private static final int ATOM_SIDX = 0x73696478;
private static final int ATOM_MOOV = 0x6D6F6F76;
private static final int ATOM_MDAT = 0x6D646174;
private static final int ATOM_MVHD = 0x6D766864;
private static final int ATOM_TRAK = 0x7472616B;
private static final int ATOM_MVEX = 0x6D766578;
private static final int ATOM_TREX = 0x74726578;
private static final int ATOM_TKHD = 0x746B6864;
private static final int ATOM_MFRA = 0x6D667261;
private static final int ATOM_TFRA = 0x74667261;
private static final int ATOM_MDHD = 0x6D646864;
private static final int BRAND_DASH = 0x64617368;
// </editor-fold>
private final DataReader stream;
private Mp4Track[] tracks = null;
private Box box;
private Moof moof;
private boolean chunkZero = false;
private int selectedTrack = -1;
public enum TrackKind {
Audio, Video, Other
}
public Mp4DashReader(SharpStream source) {
this.stream = new DataReader(source);
}
public void parse() throws IOException, NoSuchElementException {
if (selectedTrack > -1) {
return;
}
box = readBox(ATOM_FTYP);
if (parse_ftyp() != BRAND_DASH) {
throw new NoSuchElementException("Main Brand is not dash");
}
Moov moov = null;
int i;
while (box.type != ATOM_MOOF) {
ensure(box);
box = readBox();
switch (box.type) {
case ATOM_MOOV:
moov = parse_moov(box);
break;
case ATOM_SIDX:
break;
case ATOM_MFRA:
break;
case ATOM_MDAT:
throw new IOException("Expected moof, found mdat");
}
}
if (moov == null) {
throw new IOException("The provided Mp4 doesn't have the 'moov' box");
}
tracks = new Mp4Track[moov.trak.length];
for (i = 0; i < tracks.length; i++) {
tracks[i] = new Mp4Track();
tracks[i].trak = moov.trak[i];
if (moov.mvex_trex != null) {
for (Trex mvex_trex : moov.mvex_trex) {
if (tracks[i].trak.tkhd.trackId == mvex_trex.trackId) {
tracks[i].trex = mvex_trex;
}
}
}
if (moov.trak[i].tkhd.bHeight == 0 && moov.trak[i].tkhd.bWidth == 0) {
tracks[i].kind = moov.trak[i].tkhd.bVolume == 0 ? TrackKind.Other : TrackKind.Audio;
} else {
tracks[i].kind = TrackKind.Video;
}
}
}
public Mp4Track selectTrack(int index) {
selectedTrack = index;
return tracks[index];
}
/**
* Count all fragments present. This operation requires a seekable stream
*
* @return list with a basic info
* @throws IOException if the source stream is not seekeable
*/
public int getFragmentsCount() throws IOException {
if (selectedTrack < 0) {
throw new IllegalStateException("track no selected");
}
if (!stream.canRewind()) {
throw new IOException("The provided stream doesn't allow seek");
}
Box tmp;
int count = 0;
long orig_offset = stream.position();
if (box.type == ATOM_MOOF) {
tmp = box;
} else {
ensure(box);
tmp = readBox();
}
do {
if (tmp.type == ATOM_MOOF) {
ensure(readBox(ATOM_MFHD));
Box traf;
while ((traf = untilBox(tmp, ATOM_TRAF)) != null) {
Box tfhd = readBox(ATOM_TFHD);
if (parse_tfhd(tracks[selectedTrack].trak.tkhd.trackId) != null) {
count++;
break;
}
ensure(tfhd);
ensure(traf);
}
}
ensure(tmp);
} while (stream.available() && (tmp = readBox()) != null);
stream.rewind();
stream.skipBytes((int) orig_offset);
return count;
}
public Mp4Track[] getAvailableTracks() {
return tracks;
}
public Mp4TrackChunk getNextChunk() throws IOException {
Mp4Track track = tracks[selectedTrack];
while (stream.available()) {
if (chunkZero) {
ensure(box);
if (!stream.available()) {
break;
}
box = readBox();
} else {
chunkZero = true;
}
switch (box.type) {
case ATOM_MOOF:
if (moof != null) {
throw new IOException("moof found without mdat");
}
moof = parse_moof(box, track.trak.tkhd.trackId);
if (moof.traf != null) {
if (hasFlag(moof.traf.trun.bFlags, 0x0001)) {
moof.traf.trun.dataOffset -= box.size + 8;
if (moof.traf.trun.dataOffset < 0) {
throw new IOException("trun box has wrong data offset, points outside of concurrent mdat box");
}
}
if (moof.traf.trun.chunkSize < 1) {
if (hasFlag(moof.traf.tfhd.bFlags, 0x10)) {
moof.traf.trun.chunkSize = moof.traf.tfhd.defaultSampleSize * moof.traf.trun.entryCount;
} else {
moof.traf.trun.chunkSize = box.size - 8;
}
}
if (!hasFlag(moof.traf.trun.bFlags, 0x900) && moof.traf.trun.chunkDuration == 0) {
if (hasFlag(moof.traf.tfhd.bFlags, 0x20)) {
moof.traf.trun.chunkDuration = moof.traf.tfhd.defaultSampleDuration * moof.traf.trun.entryCount;
}
}
}
break;
case ATOM_MDAT:
if (moof == null) {
throw new IOException("mdat found without moof");
}
if (moof.traf == null) {
moof = null;
continue;// find another chunk
}
Mp4TrackChunk chunk = new Mp4TrackChunk();
chunk.moof = moof;
chunk.data = new TrackDataChunk(stream, moof.traf.trun.chunkSize);
moof = null;
stream.skipBytes(chunk.moof.traf.trun.dataOffset);
return chunk;
default:
}
}
return null;
}
// <editor-fold defaultState="collapsed" desc="Utils">
private long readUint() throws IOException {
return stream.readInt() & 0xffffffffL;
}
public static boolean hasFlag(int flags, int mask) {
return (flags & mask) == mask;
}
private String boxName(Box ref) {
return boxName(ref.type);
}
private String boxName(int type) {
try {
return new String(ByteBuffer.allocate(4).putInt(type).array(), "US-ASCII");
} catch (UnsupportedEncodingException e) {
return "0x" + Integer.toHexString(type);
}
}
private Box readBox() throws IOException {
Box b = new Box();
b.offset = stream.position();
b.size = stream.readInt();
b.type = stream.readInt();
return b;
}
private Box readBox(int expected) throws IOException {
Box b = readBox();
if (b.type != expected) {
throw new NoSuchElementException("expected " + boxName(expected) + " found " + boxName(b));
}
return b;
}
private void ensure(Box ref) throws IOException {
long skip = ref.offset + ref.size - stream.position();
if (skip == 0) {
return;
} else if (skip < 0) {
throw new EOFException(String.format(
"parser go beyond limits of the box. type=%s offset=%s size=%s position=%s",
boxName(ref), ref.offset, ref.size, stream.position()
));
}
stream.skipBytes((int) skip);
}
private Box untilBox(Box ref, int... expected) throws IOException {
Box b;
while (stream.position() < (ref.offset + ref.size)) {
b = readBox();
for (int type : expected) {
if (b.type == type) {
return b;
}
}
ensure(b);
}
return null;
}
// </editor-fold>
// <editor-fold defaultState="collapsed" desc="Box readers">
private Moof parse_moof(Box ref, int trackId) throws IOException {
Moof obj = new Moof();
Box b = readBox(ATOM_MFHD);
obj.mfhd_SequenceNumber = parse_mfhd();
ensure(b);
while ((b = untilBox(ref, ATOM_TRAF)) != null) {
obj.traf = parse_traf(b, trackId);
ensure(b);
if (obj.traf != null) {
return obj;
}
}
return obj;
}
private int parse_mfhd() throws IOException {
// version
// flags
stream.skipBytes(4);
return stream.readInt();
}
private Traf parse_traf(Box ref, int trackId) throws IOException {
Traf traf = new Traf();
Box b = readBox(ATOM_TFHD);
traf.tfhd = parse_tfhd(trackId);
ensure(b);
if (traf.tfhd == null) {
return null;
}
b = untilBox(ref, ATOM_TRUN, ATOM_TFDT);
if (b.type == ATOM_TFDT) {
traf.tfdt = parse_tfdt();
ensure(b);
b = readBox(ATOM_TRUN);
}
traf.trun = parse_trun();
ensure(b);
return traf;
}
private Tfhd parse_tfhd(int trackId) throws IOException {
Tfhd obj = new Tfhd();
obj.bFlags = stream.readInt();
obj.trackId = stream.readInt();
if (trackId != -1 && obj.trackId != trackId) {
return null;
}
if (hasFlag(obj.bFlags, 0x01)) {
stream.skipBytes(8);
}
if (hasFlag(obj.bFlags, 0x02)) {
stream.skipBytes(4);
}
if (hasFlag(obj.bFlags, 0x08)) {
obj.defaultSampleDuration = stream.readInt();
}
if (hasFlag(obj.bFlags, 0x10)) {
obj.defaultSampleSize = stream.readInt();
}
if (hasFlag(obj.bFlags, 0x20)) {
obj.defaultSampleFlags = stream.readInt();
}
return obj;
}
private long parse_tfdt() throws IOException {
int version = stream.read();
stream.skipBytes(3);// flags
return version == 0 ? readUint() : stream.readLong();
}
private Trun parse_trun() throws IOException {
Trun obj = new Trun();
obj.bFlags = stream.readInt();
obj.entryCount = stream.readInt();// unsigned int
obj.entries_rowSize = 0;
if (hasFlag(obj.bFlags, 0x0100)) {
obj.entries_rowSize += 4;
}
if (hasFlag(obj.bFlags, 0x0200)) {
obj.entries_rowSize += 4;
}
if (hasFlag(obj.bFlags, 0x0400)) {
obj.entries_rowSize += 4;
}
if (hasFlag(obj.bFlags, 0x0800)) {
obj.entries_rowSize += 4;
}
obj.bEntries = new byte[obj.entries_rowSize * obj.entryCount];
if (hasFlag(obj.bFlags, 0x0001)) {
obj.dataOffset = stream.readInt();
}
if (hasFlag(obj.bFlags, 0x0004)) {
obj.bFirstSampleFlags = stream.readInt();
}
stream.read(obj.bEntries);
for (int i = 0; i < obj.entryCount; i++) {
TrunEntry entry = obj.getEntry(i);
if (hasFlag(obj.bFlags, 0x0100)) {
obj.chunkDuration += entry.sampleDuration;
}
if (hasFlag(obj.bFlags, 0x0200)) {
obj.chunkSize += entry.sampleSize;
}
if (hasFlag(obj.bFlags, 0x0800)) {
if (!hasFlag(obj.bFlags, 0x0100)) {
obj.chunkDuration += entry.sampleCompositionTimeOffset;
}
}
}
return obj;
}
private int parse_ftyp() throws IOException {
int brand = stream.readInt();
stream.skipBytes(4);// minor version
return brand;
}
private Mvhd parse_mvhd() throws IOException {
int version = stream.read();
stream.skipBytes(3);// flags
// creation entries_time
// modification entries_time
stream.skipBytes(2 * (version == 0 ? 4 : 8));
Mvhd obj = new Mvhd();
obj.timeScale = readUint();
// chunkDuration
stream.skipBytes(version == 0 ? 4 : 8);
// rate
// volume
// reserved
// matrix array
// predefined
stream.skipBytes(76);
obj.nextTrackId = readUint();
return obj;
}
private Tkhd parse_tkhd() throws IOException {
int version = stream.read();
Tkhd obj = new Tkhd();
// flags
// creation entries_time
// modification entries_time
stream.skipBytes(3 + (2 * (version == 0 ? 4 : 8)));
obj.trackId = stream.readInt();
stream.skipBytes(4);// reserved
obj.duration = version == 0 ? readUint() : stream.readLong();
stream.skipBytes(2 * 4);// reserved
obj.bLayer = stream.readShort();
obj.bAlternateGroup = stream.readShort();
obj.bVolume = stream.readShort();
stream.skipBytes(2);// reserved
obj.matrix = new byte[9 * 4];
stream.read(obj.matrix);
obj.bWidth = stream.readInt();
obj.bHeight = stream.readInt();
return obj;
}
private Trak parse_trak(Box ref) throws IOException {
Trak trak = new Trak();
Box b = readBox(ATOM_TKHD);
trak.tkhd = parse_tkhd();
ensure(b);
b = untilBox(ref, ATOM_MDIA);
trak.mdia = new byte[b.size];
ByteBuffer buffer = ByteBuffer.wrap(trak.mdia);
buffer.putInt(b.size);
buffer.putInt(ATOM_MDIA);
stream.read(trak.mdia, 8, b.size - 8);
trak.mdia_mdhd_timeScale = parse_mdia(buffer);
return trak;
}
private int parse_mdia(ByteBuffer data) {
while (data.hasRemaining()) {
int end = data.position() + data.getInt();
if (data.getInt() == ATOM_MDHD) {
byte version = data.get();
data.position(data.position() + 3 + ((version == 0 ? 4 : 8) * 2));
return data.getInt();
}
data.position(end);
}
return 0;// this NEVER should happen
}
private Moov parse_moov(Box ref) throws IOException {
Box b = readBox(ATOM_MVHD);
Moov moov = new Moov();
moov.mvhd = parse_mvhd();
ensure(b);
ArrayList<Trak> tmp = new ArrayList<>((int) moov.mvhd.nextTrackId);
while ((b = untilBox(ref, ATOM_TRAK, ATOM_MVEX)) != null) {
switch (b.type) {
case ATOM_TRAK:
tmp.add(parse_trak(b));
break;
case ATOM_MVEX:
moov.mvex_trex = parse_mvex(b, (int) moov.mvhd.nextTrackId);
break;
}
ensure(b);
}
moov.trak = tmp.toArray(new Trak[tmp.size()]);
return moov;
}
private Trex[] parse_mvex(Box ref, int possibleTrackCount) throws IOException {
ArrayList<Trex> tmp = new ArrayList<>(possibleTrackCount);
Box b;
while ((b = untilBox(ref, ATOM_TREX)) != null) {
tmp.add(parse_trex());
ensure(b);
}
return tmp.toArray(new Trex[tmp.size()]);
}
private Trex parse_trex() throws IOException {
// version
// flags
stream.skipBytes(4);
Trex obj = new Trex();
obj.trackId = stream.readInt();
obj.defaultSampleDescriptionIndex = stream.readInt();
obj.defaultSampleDuration = stream.readInt();
obj.defaultSampleSize = stream.readInt();
obj.defaultSampleFlags = stream.readInt();
return obj;
}
private Tfra parse_tfra() throws IOException {
int version = stream.read();
stream.skipBytes(3);// flags
Tfra tfra = new Tfra();
tfra.trackId = stream.readInt();
stream.skipBytes(3);// reserved
int bFlags = stream.read();
int size_tts = ((bFlags >> 4) & 3) + ((bFlags >> 2) & 3) + (bFlags & 3);
tfra.entries_time = new int[stream.readInt()];
for (int i = 0; i < tfra.entries_time.length; i++) {
tfra.entries_time[i] = version == 0 ? stream.readInt() : (int) stream.readLong();
stream.skipBytes(size_tts + (version == 0 ? 4 : 8));
}
return tfra;
}
private Sidx parse_sidx() throws IOException {
int version = stream.read();
stream.skipBytes(3);// flags
Sidx obj = new Sidx();
obj.referenceId = stream.readInt();
obj.timescale = stream.readInt();
// earliest presentation entries_time
// first offset
// reserved
stream.skipBytes((2 * (version == 0 ? 4 : 8)) + 2);
obj.entries_subsegmentDuration = new int[stream.readShort()];
for (int i = 0; i < obj.entries_subsegmentDuration.length; i++) {
// reference type
// referenced size
stream.skipBytes(4);
obj.entries_subsegmentDuration[i] = stream.readInt();// unsigned int
// starts with SAP
// SAP type
// SAP delta entries_time
stream.skipBytes(4);
}
return obj;
}
private Tfra[] parse_mfra(Box ref, int trackCount) throws IOException {
ArrayList<Tfra> tmp = new ArrayList<>(trackCount);
long limit = ref.offset + ref.size;
while (stream.position() < limit) {
box = readBox();
if (box.type == ATOM_TFRA) {
tmp.add(parse_tfra());
}
ensure(box);
}
return tmp.toArray(new Tfra[tmp.size()]);
}
// </editor-fold>
// <editor-fold defaultState="collapsed" desc="Helper classes">
class Box {
int type;
long offset;
int size;
}
class Sidx {
int timescale;
int referenceId;
int[] entries_subsegmentDuration;
}
public class Moof {
int mfhd_SequenceNumber;
public Traf traf;
}
public class Traf {
public Tfhd tfhd;
long tfdt;
public Trun trun;
}
public class Tfhd {
int bFlags;
public int trackId;
int defaultSampleDuration;
int defaultSampleSize;
int defaultSampleFlags;
}
public class TrunEntry {
public int sampleDuration;
public int sampleSize;
public int sampleFlags;
public int sampleCompositionTimeOffset;
}
public class Trun {
public int chunkDuration;
public int chunkSize;
public int bFlags;
int bFirstSampleFlags;
int dataOffset;
public int entryCount;
byte[] bEntries;
int entries_rowSize;
public TrunEntry getEntry(int i) {
ByteBuffer buffer = ByteBuffer.wrap(bEntries, i * entries_rowSize, entries_rowSize);
TrunEntry entry = new TrunEntry();
if (hasFlag(bFlags, 0x0100)) {
entry.sampleDuration = buffer.getInt();
}
if (hasFlag(bFlags, 0x0200)) {
entry.sampleSize = buffer.getInt();
}
if (hasFlag(bFlags, 0x0400)) {
entry.sampleFlags = buffer.getInt();
}
if (hasFlag(bFlags, 0x0800)) {
entry.sampleCompositionTimeOffset = buffer.getInt();
}
return entry;
}
}
public class Tkhd {
int trackId;
long duration;
short bVolume;
int bWidth;
int bHeight;
byte[] matrix;
short bLayer;
short bAlternateGroup;
}
public class Trak {
public Tkhd tkhd;
public int mdia_mdhd_timeScale;
byte[] mdia;
}
class Mvhd {
long timeScale;
long nextTrackId;
}
class Moov {
Mvhd mvhd;
Trak[] trak;
Trex[] mvex_trex;
}
class Tfra {
int trackId;
int[] entries_time;
}
public class Trex {
private int trackId;
int defaultSampleDescriptionIndex;
int defaultSampleDuration;
int defaultSampleSize;
int defaultSampleFlags;
}
public class Mp4Track {
public TrackKind kind;
public Trak trak;
public Trex trex;
}
public class Mp4TrackChunk {
public InputStream data;
public Moof moof;
}
//</editor-fold>
}

623
app/src/main/java/org/schabi/newpipe/streams/Mp4DashWriter.java Normal file
View file

@ -0,0 +1,623 @@
package org.schabi.newpipe.streams;
import org.schabi.newpipe.streams.io.SharpStream;
import org.schabi.newpipe.streams.Mp4DashReader.Mp4Track;
import org.schabi.newpipe.streams.Mp4DashReader.Mp4TrackChunk;
import org.schabi.newpipe.streams.Mp4DashReader.Trak;
import org.schabi.newpipe.streams.Mp4DashReader.Trex;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import static org.schabi.newpipe.streams.Mp4DashReader.hasFlag;
/**
*
* @author kapodamy
*/
public class Mp4DashWriter {
private final static byte DIMENSIONAL_FIVE = 5;
private final static byte DIMENSIONAL_TWO = 2;
private final static short DEFAULT_TIMESCALE = 1000;
private final static int BUFFER_SIZE = 8 * 1024;
private final static byte DEFAULT_TREX_SIZE = 32;
private final static byte[] TFRA_TTS_DEFAULT = new byte[]{0x01, 0x01, 0x01};
private final static int EPOCH_OFFSET = 2082844800;
private Mp4Track[] infoTracks;
private SharpStream[] sourceTracks;
private Mp4DashReader[] readers;
private final long time;
private boolean done = false;
private boolean parsed = false;
private long written = 0;
private ArrayList<ArrayList<Integer>> chunkTimes;
private ArrayList<Long> moofOffsets;
private ArrayList<Integer> fragSizes;
public Mp4DashWriter(SharpStream... source) {
sourceTracks = source;
readers = new Mp4DashReader[sourceTracks.length];
infoTracks = new Mp4Track[sourceTracks.length];
time = (System.currentTimeMillis() / 1000L) + EPOCH_OFFSET;
}
public Mp4Track[] getTracksFromSource(int sourceIndex) throws IllegalStateException {
if (!parsed) {
throw new IllegalStateException("All sources must be parsed first");
}
return readers[sourceIndex].getAvailableTracks();
}
public void parseSources() throws IOException, IllegalStateException {
if (done) {
throw new IllegalStateException("already done");
}
if (parsed) {
throw new IllegalStateException("already parsed");
}
try {
for (int i = 0; i < readers.length; i++) {
readers[i] = new Mp4DashReader(sourceTracks[i]);
readers[i].parse();
}
} finally {
parsed = true;
}
}
public void selectTracks(int... trackIndex) throws IOException {
if (done) {
throw new IOException("already done");
}
if (chunkTimes != null) {
throw new IOException("tracks already selected");
}
try {
chunkTimes = new ArrayList<>(readers.length);
moofOffsets = new ArrayList<>(32);
fragSizes = new ArrayList<>(32);
for (int i = 0; i < readers.length; i++) {
infoTracks[i] = readers[i].selectTrack(trackIndex[i]);
chunkTimes.add(new ArrayList<Integer>(32));
}
} finally {
parsed = true;
}
}
public long getBytesWritten() {
return written;
}
public void build(SharpStream out) throws IOException, RuntimeException {
if (done) {
throw new RuntimeException("already done");
}
if (!out.canWrite()) {
throw new IOException("the provided output is not writable");
}
long sidxOffsets = -1;
int maxFrags = 0;
for (SharpStream stream : sourceTracks) {
if (!stream.canRewind()) {
sidxOffsets = -2;// sidx not available
}
}
try {
dump(make_ftyp(), out);
dump(make_moov(), out);
if (sidxOffsets == -1 && out.canRewind()) {
//<editor-fold defaultstate="collapsed" desc="calculate sidx">
int reserved = 0;
for (Mp4DashReader reader : readers) {
int count = reader.getFragmentsCount();
if (count > maxFrags) {
maxFrags = count;
}
reserved += 12 + calcSidxBodySize(count);
}
if (maxFrags > 0xFFFF) {
sidxOffsets = -3;// TODO: to many fragments, needs a multi-sidx implementation
} else {
sidxOffsets = written;
dump(make_free(reserved), out);
}
//</editor-fold>
}
ArrayList<Mp4TrackChunk> chunks = new ArrayList<>(readers.length);
chunks.add(null);
int read;
byte[] buffer = new byte[BUFFER_SIZE];
int sequenceNumber = 1;
while (true) {
chunks.clear();
for (int i = 0; i < readers.length; i++) {
Mp4TrackChunk chunk = readers[i].getNextChunk();
if (chunk == null || chunk.moof.traf.trun.chunkSize < 1) {
continue;
}
chunk.moof.traf.tfhd.trackId = i + 1;
chunks.add(chunk);
if (sequenceNumber == 1) {
if (chunk.moof.traf.trun.entryCount > 0 && hasFlag(chunk.moof.traf.trun.bFlags, 0x0800)) {
chunkTimes.get(i).add(chunk.moof.traf.trun.getEntry(0).sampleCompositionTimeOffset);
} else {
chunkTimes.get(i).add(0);
}
}
chunkTimes.get(i).add(chunk.moof.traf.trun.chunkDuration);
}
if (chunks.size() < 1) {
break;
}
long offset = written;
moofOffsets.add(offset);
dump(make_moof(sequenceNumber++, chunks, offset), out);
dump(make_mdat(chunks), out);
for (Mp4TrackChunk chunk : chunks) {
while ((read = chunk.data.read(buffer)) > 0) {
out.write(buffer, 0, read);
written += read;
}
}
fragSizes.add((int) (written - offset));
}
dump(make_mfra(), out);
if (sidxOffsets > 0 && moofOffsets.size() == maxFrags) {
long len = written;
out.rewind();
out.skip(sidxOffsets);
written = sidxOffsets;
sidxOffsets = moofOffsets.get(0);
for (int i = 0; i < readers.length; i++) {
dump(make_sidx(i, sidxOffsets - written), out);
}
written = len;
}
} finally {
done = true;
}
}
public boolean isDone() {
return done;
}
public boolean isParsed() {
return parsed;
}
public void close() {
done = true;
parsed = true;
for (SharpStream src : sourceTracks) {
src.dispose();
}
sourceTracks = null;
readers = null;
infoTracks = null;
moofOffsets = null;
chunkTimes = null;
}
// <editor-fold defaultstate="collapsed" desc="Utils">
private void dump(byte[][] buffer, SharpStream stream) throws IOException {
for (byte[] buff : buffer) {
stream.write(buff);
written += buff.length;
}
}
private byte[][] lengthFor(byte[][] buffer) {
int length = 0;
for (byte[] buff : buffer) {
length += buff.length;
}
ByteBuffer.wrap(buffer[0]).putInt(length);
return buffer;
}
private int calcSidxBodySize(int entryCount) {
return 4 + 4 + 8 + 8 + 4 + (entryCount * 12);
}
// </editor-fold>
// <editor-fold defaultstate="collapsed" desc="Box makers">
private byte[][] make_moof(int sequence, ArrayList<Mp4TrackChunk> chunks, long referenceOffset) {
int pos = 2;
TrunExtra[] extra = new TrunExtra[chunks.size()];
byte[][] buffer = new byte[pos + (extra.length * DIMENSIONAL_FIVE)][];
buffer[0] = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x6D, 0x6F, 0x6F, 0x66,// info header
0x00, 0x00, 0x00, 0x10, 0x6D, 0x66, 0x68, 0x64, 0x00, 0x00, 0x00, 0x00//mfhd
};
buffer[1] = new byte[4];
ByteBuffer.wrap(buffer[1]).putInt(sequence);
for (int i = 0; i < extra.length; i++) {
extra[i] = new TrunExtra();
for (byte[] buff : make_traf(chunks.get(i), extra[i], referenceOffset)) {
buffer[pos++] = buff;
}
}
lengthFor(buffer);
int offset = 8 + ByteBuffer.wrap(buffer[0]).getInt();
for (int i = 0; i < extra.length; i++) {
extra[i].byteBuffer.putInt(offset);
offset += chunks.get(i).moof.traf.trun.chunkSize;
}
return buffer;
}
private byte[][] make_traf(Mp4TrackChunk chunk, TrunExtra extra, long moofOffset) {
byte[][] buffer = new byte[DIMENSIONAL_FIVE][];
buffer[0] = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x61, 0x66,
0x00, 0x00, 0x00, 0x00, 0x74, 0x66, 0x68, 0x64
};
int flags = (chunk.moof.traf.tfhd.bFlags & 0x38) | 0x01;
byte tfhdBodySize = 8 + 8;
if (hasFlag(flags, 0x08)) {
tfhdBodySize += 4;
}
if (hasFlag(flags, 0x10)) {
tfhdBodySize += 4;
}
if (hasFlag(flags, 0x20)) {
tfhdBodySize += 4;
}
buffer[1] = new byte[tfhdBodySize];
ByteBuffer set = ByteBuffer.wrap(buffer[1]);
set.position(4);
set.putInt(chunk.moof.traf.tfhd.trackId);
set.putLong(moofOffset);
if (hasFlag(flags, 0x08)) {
set.putInt(chunk.moof.traf.tfhd.defaultSampleDuration);
}
if (hasFlag(flags, 0x10)) {
set.putInt(chunk.moof.traf.tfhd.defaultSampleSize);
}
if (hasFlag(flags, 0x20)) {
set.putInt(chunk.moof.traf.tfhd.defaultSampleFlags);
}
set.putInt(0, flags);
ByteBuffer.wrap(buffer[0]).putInt(8, 8 + tfhdBodySize);
buffer[2] = new byte[]{
0x00, 0x00, 0x00, 0x14,
0x74, 0x66, 0x64, 0x74,
0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00
};
ByteBuffer.wrap(buffer[2]).putLong(12, chunk.moof.traf.tfdt);
buffer[3] = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x75, 0x6E,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00
};
buffer[4] = chunk.moof.traf.trun.bEntries;
lengthFor(buffer);
set = ByteBuffer.wrap(buffer[3]);
set.putInt(buffer[3].length + buffer[4].length);
set.position(8);
set.putInt((chunk.moof.traf.trun.bFlags | 0x01) & 0x0F01);
set.putInt(chunk.moof.traf.trun.entryCount);
extra.byteBuffer = set;
return buffer;
}
private byte[][] make_mdat(ArrayList<Mp4TrackChunk> chunks) {
byte[][] buffer = new byte[][]{
{
0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x61, 0x74
}
};
int length = 0;
for (Mp4TrackChunk chunk : chunks) {
length += chunk.moof.traf.trun.chunkSize;
}
ByteBuffer.wrap(buffer[0]).putInt(length + 8);
return buffer;
}
private byte[][] make_ftyp() {
return new byte[][]{
{
0x00, 0x00, 0x00, 0x20, 0x66, 0x74, 0x79, 0x70, 0x64, 0x61, 0x73, 0x68, 0x00, 0x00, 0x00, 0x00,
0x6D, 0x70, 0x34, 0x31, 0x69, 0x73, 0x6F, 0x6D, 0x69, 0x73, 0x6F, 0x36, 0x69, 0x73, 0x6F, 0x32
}
};
}
private byte[][] make_mvhd() {
byte[][] buffer = new byte[DIMENSIONAL_FIVE][];
buffer[0] = new byte[]{
0x00, 0x00, 0x00, 0x78, 0x6D, 0x76, 0x68, 0x64, 0x01, 0x00, 0x00, 0x00
};
buffer[1] = new byte[28];
buffer[2] = new byte[]{
0x00, 0x01, 0x00, 0x00, 0x01, 0x00,// default volume and rate
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,// reserved values
// default matrix
0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x40, 0x00, 0x00, 0x00
};
buffer[3] = new byte[24];// predefined
buffer[4] = ByteBuffer.allocate(4).putInt(infoTracks.length + 1).array();
long longestTrack = 0;
for (Mp4Track track : infoTracks) {
long tmp = (long) ((track.trak.tkhd.duration / (double) track.trak.mdia_mdhd_timeScale) * DEFAULT_TIMESCALE);
if (tmp > longestTrack) {
longestTrack = tmp;
}
}
ByteBuffer.wrap(buffer[1])
.putLong(time)
.putLong(time)
.putInt(DEFAULT_TIMESCALE)
.putLong(longestTrack);
return buffer;
}
private byte[][] make_trak(int trackId, Trak trak) throws RuntimeException {
if (trak.tkhd.matrix.length != 36) {
throw new RuntimeException("bad track matrix length (expected 36)");
}
byte[][] buffer = new byte[DIMENSIONAL_FIVE][];
buffer[0] = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x74, 0x72, 0x61, 0x6B,// trak header
0x00, 0x00, 0x00, 0x68, 0x74, 0x6B, 0x68, 0x64, 0x01, 0x00, 0x00, 0x03 // tkhd header
};
buffer[1] = new byte[48];
buffer[2] = trak.tkhd.matrix;
buffer[3] = new byte[8];
buffer[4] = trak.mdia;
ByteBuffer set = ByteBuffer.wrap(buffer[1]);
set.putLong(time);
set.putLong(time);
set.putInt(trackId);
set.position(24);
set.putLong(trak.tkhd.duration);
set.position(40);
set.putShort(trak.tkhd.bLayer);
set.putShort(trak.tkhd.bAlternateGroup);
set.putShort(trak.tkhd.bVolume);
ByteBuffer.wrap(buffer[3])
.putInt(trak.tkhd.bWidth)
.putInt(trak.tkhd.bHeight);
return lengthFor(buffer);
}
private byte[][] make_moov() throws RuntimeException {
int pos = 1;
byte[][] buffer = new byte[2 + (DIMENSIONAL_TWO * infoTracks.length) + (DIMENSIONAL_FIVE * infoTracks.length) + DIMENSIONAL_FIVE + 1][];
buffer[0] = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x6D, 0x6F, 0x6F, 0x76
};
for (byte[] buff : make_mvhd()) {
buffer[pos++] = buff;
}
for (int i = 0; i < infoTracks.length; i++) {
for (byte[] buff : make_trak(i + 1, infoTracks[i].trak)) {
buffer[pos++] = buff;
}
}
buffer[pos] = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x6D, 0x76, 0x65, 0x78
};
ByteBuffer.wrap(buffer[pos++]).putInt((infoTracks.length * DEFAULT_TREX_SIZE) + 8);
for (int i = 0; i < infoTracks.length; i++) {
for (byte[] buff : make_trex(i + 1, infoTracks[i].trex)) {
buffer[pos++] = buff;
}
}
// default udta
buffer[pos] = new byte[]{
0x00, 0x00, 0x00, 0x5C, 0x75, 0x64, 0x74, 0x61, 0x00, 0x00, 0x00, 0x54, 0x6D, 0x65, 0x74, 0x61,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x68, 0x64, 0x6C, 0x72, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x6D, 0x64, 0x69, 0x72, 0x61, 0x70, 0x70, 0x6C, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x69, 0x6C, 0x73, 0x74, 0x00, 0x00, 0x00,
0x1F, (byte) 0xA9, 0x63, 0x6D, 0x74, 0x00, 0x00, 0x00, 0x17, 0x64, 0x61, 0x74, 0x61, 0x00, 0x00, 0x00,
0x01, 0x00, 0x00, 0x00, 0x00,
0x4E, 0x65, 0x77, 0x50, 0x69, 0x70, 0x65// "NewPipe" binary string
};
return lengthFor(buffer);
}
private byte[][] make_trex(int trackId, Trex trex) {
byte[][] buffer = new byte[][]{
{
0x00, 0x00, 0x00, 0x20, 0x74, 0x72, 0x65, 0x78, 0x00, 0x00, 0x00, 0x00
},
new byte[20]
};
ByteBuffer.wrap(buffer[1])
.putInt(trackId)
.putInt(trex.defaultSampleDescriptionIndex)
.putInt(trex.defaultSampleDuration)
.putInt(trex.defaultSampleSize)
.putInt(trex.defaultSampleFlags);
return buffer;
}
private byte[][] make_tfra(int trackId, List<Integer> times, List<Long> moofOffsets) {
int entryCount = times.size() - 1;
byte[][] buffer = new byte[DIMENSIONAL_TWO][];
buffer[0] = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x74, 0x66, 0x72, 0x61, 0x01, 0x00, 0x00, 0x00
};
buffer[1] = new byte[12 + ((16 + TFRA_TTS_DEFAULT.length) * entryCount)];
ByteBuffer set = ByteBuffer.wrap(buffer[1]);
set.putInt(trackId);
set.position(8);
set.putInt(entryCount);
long decodeTime = 0;
for (int i = 0; i < entryCount; i++) {
decodeTime += times.get(i);
set.putLong(decodeTime);
set.putLong(moofOffsets.get(i));
set.put(TFRA_TTS_DEFAULT);// default values: traf number/trun number/sample number
}
return lengthFor(buffer);
}
private byte[][] make_mfra() {
byte[][] buffer = new byte[2 + (DIMENSIONAL_TWO * infoTracks.length)][];
buffer[0] = new byte[]{
0x00, 0x00, 0x00, 0x00, 0x6D, 0x66, 0x72, 0x61
};
int pos = 1;
for (int i = 0; i < infoTracks.length; i++) {
for (byte[] buff : make_tfra(i + 1, chunkTimes.get(i), moofOffsets)) {
buffer[pos++] = buff;
}
}
buffer[pos] = new byte[]{// mfro
0x00, 0x00, 0x00, 0x10, 0x6D, 0x66, 0x72, 0x6F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
lengthFor(buffer);
ByteBuffer set = ByteBuffer.wrap(buffer[pos]);
set.position(12);
set.put(buffer[0], 0, 4);
return buffer;
}
private byte[][] make_sidx(int internalTrackId, long firstOffset) {
List<Integer> times = chunkTimes.get(internalTrackId);
int count = times.size() - 1;// the first item is ignored (composition time)
if (count > 65535) {
throw new OutOfMemoryError("to many fragments. sidx limit is 65535, found " + String.valueOf(count));
}
byte[][] buffer = new byte[][]{
new byte[]{
0x00, 0x00, 0x00, 0x00, 0x73, 0x69, 0x64, 0x78, 0x01, 0x00, 0x00, 0x00
},
new byte[calcSidxBodySize(count)]
};
lengthFor(buffer);
ByteBuffer set = ByteBuffer.wrap(buffer[1]);
set.putInt(internalTrackId + 1);
set.putInt(infoTracks[internalTrackId].trak.mdia_mdhd_timeScale);
set.putLong(0);
set.putLong(firstOffset - ByteBuffer.wrap(buffer[0]).getInt());
set.putInt(0xFFFF & count);// unsigned
int i = 0;
while (i < count) {
set.putInt(fragSizes.get(i) & 0x7fffffff);// default reference type is 0
set.putInt(times.get(i + 1));
set.putInt(0x90000000);// default SAP settings
i++;
}
return buffer;
}
private byte[][] make_free(int totalSize) {
return lengthFor(new byte[][]{
new byte[]{0x00, 0x00, 0x00, 0x00, 0x66, 0x72, 0x65, 0x65},
new byte[totalSize - 8]// this is waste of RAM
});
}
//</editor-fold>
class TrunExtra {
ByteBuffer byteBuffer;
}
}

370
app/src/main/java/org/schabi/newpipe/streams/SubtitleConverter.java Normal file
View file

@ -0,0 +1,370 @@
package org.schabi.newpipe.streams;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.Charset;
import java.text.ParseException;
import java.util.Locale;
import org.schabi.newpipe.streams.io.SharpStream;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
/**
* @author kapodamy
*/
public class SubtitleConverter {
private static final String NEW_LINE = "\r\n";
public void dumpTTML(SharpStream in, final SharpStream out, final boolean ignoreEmptyFrames, final boolean detectYoutubeDuplicateLines
) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException {
final FrameWriter callback = new FrameWriter() {
int frameIndex = 0;
final Charset charset = Charset.forName("utf-8");
@Override
public void yield(SubtitleFrame frame) throws IOException {
if (ignoreEmptyFrames && frame.isEmptyText()) {
return;
}
out.write(String.valueOf(frameIndex++).getBytes(charset));
out.write(NEW_LINE.getBytes(charset));
out.write(getTime(frame.start, true).getBytes(charset));
out.write(" --> ".getBytes(charset));
out.write(getTime(frame.end, true).getBytes(charset));
out.write(NEW_LINE.getBytes(charset));
out.write(frame.text.getBytes(charset));
out.write(NEW_LINE.getBytes(charset));
out.write(NEW_LINE.getBytes(charset));
}
};
read_xml_based(in, callback, detectYoutubeDuplicateLines,
"tt", "xmlns", "http://www.w3.org/ns/ttml",
new String[]{"timedtext", "head", "wp"},
new String[]{"body", "div", "p"},
"begin", "end", true
);
}
private void read_xml_based(SharpStream source, FrameWriter callback, boolean detectYoutubeDuplicateLines,
String root, String formatAttr, String formatVersion, String[] cuePath, String[] framePath,
String timeAttr, String durationAttr, boolean hasTimestamp
) throws IOException, ParseException, SAXException, ParserConfigurationException, XPathExpressionException {
/*
* XML based subtitles parser with BASIC support
* multiple CUE is not supported
* styling is not supported
* tag timestamps (in auto-generated subtitles) are not supported, maybe in the future
* also TimestampTagOption enum is not applicable
* Language parsing is not supported
*/
byte[] buffer = new byte[source.available()];
source.read(buffer);
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true);
DocumentBuilder builder = factory.newDocumentBuilder();
Document xml = builder.parse(new ByteArrayInputStream(buffer));
String attr;
// get the format version or namespace
Element node = xml.getDocumentElement();
if (node == null) {
throw new ParseException("Can't get the format version. ¿wrong namespace?", -1);
} else if (!node.getNodeName().equals(root)) {
throw new ParseException("Invalid root", -1);
}
if (formatAttr.equals("xmlns")) {
if (!node.getNamespaceURI().equals(formatVersion)) {
throw new UnsupportedOperationException("Expected xml namespace: " + formatVersion);
}
} else {
attr = node.getAttributeNS(formatVersion, formatAttr);
if (attr == null) {
throw new ParseException("Can't get the format attribute", -1);
}
if (!attr.equals(formatVersion)) {
throw new ParseException("Invalid format version : " + attr, -1);
}
}
NodeList node_list;
int line_break = 0;// Maximum characters per line if present (valid for TranScript v3)
if (!hasTimestamp) {
node_list = selectNodes(xml, cuePath, formatVersion);
if (node_list != null) {
// if the subtitle has multiple CUEs, use the highest value
for (int i = 0; i < node_list.getLength(); i++) {
try {
int tmp = Integer.parseInt(((Element) node_list.item(i)).getAttributeNS(formatVersion, "ah"));
if (tmp > line_break) {
line_break = tmp;
}
} catch (Exception err) {
}
}
}
}
// parse every frame
node_list = selectNodes(xml, framePath, formatVersion);
if (node_list == null) {
return;// no frames detected
}
int fs_ff = -1;// first timestamp of first frame
boolean limit_lines = false;
for (int i = 0; i < node_list.getLength(); i++) {
Element elem = (Element) node_list.item(i);
SubtitleFrame obj = new SubtitleFrame();
obj.text = elem.getTextContent();
attr = elem.getAttribute(timeAttr);// ¡this cant be null!
obj.start = hasTimestamp ? parseTimestamp(attr) : Integer.parseInt(attr);
attr = elem.getAttribute(durationAttr);
if (obj.text == null || attr == null) {
continue;// normally is a blank line (on auto-generated subtitles) ignore
}
if (hasTimestamp) {
obj.end = parseTimestamp(attr);
if (detectYoutubeDuplicateLines) {
if (limit_lines) {
int swap = obj.end;
obj.end = fs_ff;
fs_ff = swap;
} else {
if (fs_ff < 0) {
fs_ff = obj.end;
} else {
if (fs_ff < obj.start) {
limit_lines = true;// the subtitles has duplicated lines
} else {
detectYoutubeDuplicateLines = false;
}
}
}
}
} else {
obj.end = obj.start + Integer.parseInt(attr);
}
if (/*node.getAttribute("w").equals("1") &&*/line_break > 1 && obj.text.length() > line_break) {
// implement auto line breaking (once)
StringBuilder text = new StringBuilder(obj.text);
obj.text = null;
switch (text.charAt(line_break)) {
case ' ':
case '\t':
putBreakAt(line_break, text);
break;
default:// find the word start position
for (int j = line_break - 1; j > 0; j--) {
switch (text.charAt(j)) {
case ' ':
case '\t':
putBreakAt(j, text);
j = -1;
break;
case '\r':
case '\n':
j = -1;// long word, just ignore
break;
}
}
break;
}
obj.text = text.toString();// set the processed text
}
callback.yield(obj);
}
}
private static NodeList selectNodes(Document xml, String[] path, String namespaceUri) throws XPathExpressionException {
Element ref = xml.getDocumentElement();
for (int i = 0; i < path.length - 1; i++) {
NodeList nodes = ref.getChildNodes();
if (nodes.getLength() < 1) {
return null;
}
Element elem;
for (int j = 0; j < nodes.getLength(); j++) {
if (nodes.item(j).getNodeType() == Node.ELEMENT_NODE) {
elem = (Element) nodes.item(j);
if (elem.getNodeName().equals(path[i]) && elem.getNamespaceURI().equals(namespaceUri)) {
ref = elem;
break;
}
}
}
}
return ref.getElementsByTagNameNS(namespaceUri, path[path.length - 1]);
}
private static int parseTimestamp(String multiImpl) throws NumberFormatException, ParseException {
if (multiImpl.length() < 1) {
return 0;
} else if (multiImpl.length() == 1) {
return Integer.parseInt(multiImpl) * 1000;// ¡this must be a number in seconds!
}
// detect wallclock-time
if (multiImpl.startsWith("wallclock(")) {
throw new UnsupportedOperationException("Parsing wallclock timestamp is not implemented");
}
// detect offset-time
if (multiImpl.indexOf(':') < 0) {
int multiplier = 1000;
char metric = multiImpl.charAt(multiImpl.length() - 1);
switch (metric) {
case 'h':
multiplier *= 3600000;
break;
case 'm':
multiplier *= 60000;
break;
case 's':
if (multiImpl.charAt(multiImpl.length() - 2) == 'm') {
multiplier = 1;// ms
}
break;
default:
if (!Character.isDigit(metric)) {
throw new NumberFormatException("Invalid metric suffix found on : " + multiImpl);
}
metric = '\0';
break;
}
try {
String offset_time = multiImpl;
if (multiplier == 1) {
offset_time = offset_time.substring(0, offset_time.length() - 2);
} else if (metric != '\0') {
offset_time = offset_time.substring(0, offset_time.length() - 1);
}
double time_metric_based = Double.parseDouble(offset_time);
if (Math.abs(time_metric_based) <= Double.MAX_VALUE) {
return (int) (time_metric_based * multiplier);
}
} catch (Exception err) {
throw new UnsupportedOperationException("Invalid or not implemented timestamp on: " + multiImpl);
}
}
// detect clock-time
int time = 0;
String[] units = multiImpl.split(":");
if (units.length < 3) {
throw new ParseException("Invalid clock-time timestamp", -1);
}
time += Integer.parseInt(units[0]) * 3600000;// hours
time += Integer.parseInt(units[1]) * 60000;//minutes
time += Float.parseFloat(units[2]) * 1000f;// seconds and milliseconds (if present)
// frames and sub-frames are ignored (not implemented)
// time += units[3] * fps;
return time;
}
private static void putBreakAt(int idx, StringBuilder str) {
// this should be optimized at compile time
if (NEW_LINE.length() > 1) {
str.delete(idx, idx + 1);// remove after replace
str.insert(idx, NEW_LINE);
} else {
str.setCharAt(idx, NEW_LINE.charAt(0));
}
}
private static String getTime(int time, boolean comma) {
// cast every value to integer to avoid auto-round in ToString("00").
StringBuilder str = new StringBuilder(12);
str.append(numberToString(time / 1000 / 3600, 2));// hours
str.append(':');
str.append(numberToString(time / 1000 / 60 % 60, 2));// minutes
str.append(':');
str.append(numberToString(time / 1000 % 60, 2));// seconds
str.append(comma ? ',' : '.');
str.append(numberToString(time % 1000, 3));// miliseconds
return str.toString();
}
private static String numberToString(int nro, int pad) {
return String.format(Locale.ENGLISH, "%0".concat(String.valueOf(pad)).concat("d"), nro);
}
/******************
* helper classes *
******************/
private interface FrameWriter {
void yield(SubtitleFrame frame) throws IOException;
}
private static class SubtitleFrame {
//Java no support unsigned int
public int end;
public int start;
public String text = "";
private boolean isEmptyText() {
if (text == null) {
return true;
}
for (int i = 0; i < text.length(); i++) {
switch (text.charAt(i)) {
case ' ':
case '\t':
case '\r':
case '\n':
break;
default:
return false;
}
}
return true;
}
}
}

65
app/src/main/java/org/schabi/newpipe/streams/TrackDataChunk.java Normal file
View file

@ -0,0 +1,65 @@
package org.schabi.newpipe.streams;
import java.io.InputStream;
import java.io.IOException;
public class TrackDataChunk extends InputStream {
private final DataReader base;
private int size;
public TrackDataChunk(DataReader base, int size) {
this.base = base;
this.size = size;
}
@Override
public int read() throws IOException {
if (size < 1) {
return -1;
}
int res = base.read();
if (res >= 0) {
size--;
}
return res;
}
@Override
public int read(byte[] buffer) throws IOException {
return read(buffer, 0, buffer.length);
}
@Override
public int read(byte[] buffer, int offset, int count) throws IOException {
count = Math.min(size, count);
int read = base.read(buffer, offset, count);
size -= count;
return read;
}
@Override
public long skip(long amount) throws IOException {
long res = base.skipBytes(Math.min(amount, size));
size -= res;
return res;
}
@Override
public int available() {
return size;
}
@Override
public void close() {
size = 0;
}
@Override
public boolean markSupported() {
return false;
}
}

507
app/src/main/java/org/schabi/newpipe/streams/WebMReader.java Normal file
View file

@ -0,0 +1,507 @@
package org.schabi.newpipe.streams;
import java.io.EOFException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.NoSuchElementException;
import java.util.Objects;
import org.schabi.newpipe.streams.io.SharpStream;
/**
*
* @author kapodamy
*/
public class WebMReader {
//<editor-fold defaultState="collapsed" desc="constants">
private final static int ID_EMBL = 0x0A45DFA3;
private final static int ID_EMBLReadVersion = 0x02F7;
private final static int ID_EMBLDocType = 0x0282;
private final static int ID_EMBLDocTypeReadVersion = 0x0285;
private final static int ID_Segment = 0x08538067;
private final static int ID_Info = 0x0549A966;
private final static int ID_TimecodeScale = 0x0AD7B1;
private final static int ID_Duration = 0x489;
private final static int ID_Tracks = 0x0654AE6B;
private final static int ID_TrackEntry = 0x2E;
private final static int ID_TrackNumber = 0x57;
private final static int ID_TrackType = 0x03;
private final static int ID_CodecID = 0x06;
private final static int ID_CodecPrivate = 0x23A2;
private final static int ID_Video = 0x60;
private final static int ID_Audio = 0x61;
private final static int ID_DefaultDuration = 0x3E383;
private final static int ID_FlagLacing = 0x1C;
private final static int ID_Cluster = 0x0F43B675;
private final static int ID_Timecode = 0x67;
private final static int ID_SimpleBlock = 0x23;
//</editor-fold>
public enum TrackKind {
Audio/*2*/, Video/*1*/, Other
}
private DataReader stream;
private Segment segment;
private WebMTrack[] tracks;
private int selectedTrack;
private boolean done;
private boolean firstSegment;
public WebMReader(SharpStream source) {
this.stream = new DataReader(source);
}
public void parse() throws IOException {
Element elem = readElement(ID_EMBL);
if (!readEbml(elem, 1, 2)) {
throw new UnsupportedOperationException("Unsupported EBML data (WebM)");
}
ensure(elem);
elem = untilElement(null, ID_Segment);
if (elem == null) {
throw new IOException("Fragment element not found");
}
segment = readSegment(elem, 0, true);
tracks = segment.tracks;
selectedTrack = -1;
done = false;
firstSegment = true;
}
public WebMTrack[] getAvailableTracks() {
return tracks;
}
public WebMTrack selectTrack(int index) {
selectedTrack = index;
return tracks[index];
}
public Segment getNextSegment() throws IOException {
if (done) {
return null;
}
if (firstSegment && segment != null) {
firstSegment = false;
return segment;
}
ensure(segment.ref);
Element elem = untilElement(null, ID_Segment);
if (elem == null) {
done = true;
return null;
}
segment = readSegment(elem, 0, false);
return segment;
}
//<editor-fold defaultstate="collapsed" desc="utils">
private long readNumber(Element parent) throws IOException {
int length = (int) parent.contentSize;
long value = 0;
while (length-- > 0) {
int read = stream.read();
if (read == -1) {
throw new EOFException();
}
value = (value << 8) | read;
}
return value;
}
private String readString(Element parent) throws IOException {
return new String(readBlob(parent), "utf-8");
}
private byte[] readBlob(Element parent) throws IOException {
long length = parent.contentSize;
byte[] buffer = new byte[(int) length];
int read = stream.read(buffer);
if (read < length) {
throw new EOFException();
}
return buffer;
}
private long readEncodedNumber() throws IOException {
int value = stream.read();
if (value > 0) {
byte size = 1;
int mask = 0x80;
while (size < 9) {
if ((value & mask) == mask) {
mask = 0xFF;
mask >>= size;
long number = value & mask;
for (int i = 1; i < size; i++) {
value = stream.read();
number <<= 8;
number |= value;
}
return number;
}
mask >>= 1;
size++;
}
}
throw new IOException("Invalid encoded length");
}
private Element readElement() throws IOException {
Element elem = new Element();
elem.offset = stream.position();
elem.type = (int) readEncodedNumber();
elem.contentSize = readEncodedNumber();
elem.size = elem.contentSize + stream.position() - elem.offset;
return elem;
}
private Element readElement(int expected) throws IOException {
Element elem = readElement();
if (expected != 0 && elem.type != expected) {
throw new NoSuchElementException("expected " + elementID(expected) + " found " + elementID(elem.type));
}
return elem;
}
private Element untilElement(Element ref, int... expected) throws IOException {
Element elem;
while (ref == null ? stream.available() : (stream.position() < (ref.offset + ref.size))) {
elem = readElement();
for (int type : expected) {
if (elem.type == type) {
return elem;
}
}
ensure(elem);
}
return null;
}
private String elementID(long type) {
return "0x".concat(Long.toHexString(type));
}
private void ensure(Element ref) throws IOException {
long skip = (ref.offset + ref.size) - stream.position();
if (skip == 0) {
return;
} else if (skip < 0) {
throw new EOFException(String.format(
"parser go beyond limits of the Element. type=%s offset=%s size=%s position=%s",
elementID(ref.type), ref.offset, ref.size, stream.position()
));
}
stream.skipBytes(skip);
}
//</editor-fold>
//<editor-fold defaultState="collapsed" desc="elements readers">
private boolean readEbml(Element ref, int minReadVersion, int minDocTypeVersion) throws IOException {
Element elem = untilElement(ref, ID_EMBLReadVersion);
if (elem == null) {
return false;
}
if (readNumber(elem) > minReadVersion) {
return false;
}
elem = untilElement(ref, ID_EMBLDocType);
if (elem == null) {
return false;
}
if (!readString(elem).equals("webm")) {
return false;
}
elem = untilElement(ref, ID_EMBLDocTypeReadVersion);
return elem != null && readNumber(elem) <= minDocTypeVersion;
}
private Info readInfo(Element ref) throws IOException {
Element elem;
Info info = new Info();
while ((elem = untilElement(ref, ID_TimecodeScale, ID_Duration)) != null) {
switch (elem.type) {
case ID_TimecodeScale:
info.timecodeScale = readNumber(elem);
break;
case ID_Duration:
info.duration = readNumber(elem);
break;
}
ensure(elem);
}
if (info.timecodeScale == 0) {
throw new NoSuchElementException("Element Timecode not found");
}
return info;
}
private Segment readSegment(Element ref, int trackLacingExpected, boolean metadataExpected) throws IOException {
Segment obj = new Segment(ref);
Element elem;
while ((elem = untilElement(ref, ID_Info, ID_Tracks, ID_Cluster)) != null) {
if (elem.type == ID_Cluster) {
obj.currentCluster = elem;
break;
}
switch (elem.type) {
case ID_Info:
obj.info = readInfo(elem);
break;
case ID_Tracks:
obj.tracks = readTracks(elem, trackLacingExpected);
break;
}
ensure(elem);
}
if (metadataExpected && (obj.info == null || obj.tracks == null)) {
throw new RuntimeException("Cluster element found without Info and/or Tracks element at position " + String.valueOf(ref.offset));
}
return obj;
}
private WebMTrack[] readTracks(Element ref, int lacingExpected) throws IOException {
ArrayList<WebMTrack> trackEntries = new ArrayList<>(2);
Element elem_trackEntry;
while ((elem_trackEntry = untilElement(ref, ID_TrackEntry)) != null) {
WebMTrack entry = new WebMTrack();
boolean drop = false;
Element elem;
while ((elem = untilElement(elem_trackEntry,
ID_TrackNumber, ID_TrackType, ID_CodecID, ID_CodecPrivate, ID_FlagLacing, ID_DefaultDuration, ID_Audio, ID_Video
)) != null) {
switch (elem.type) {
case ID_TrackNumber:
entry.trackNumber = readNumber(elem);
break;
case ID_TrackType:
entry.trackType = (int)readNumber(elem);
break;
case ID_CodecID:
entry.codecId = readString(elem);
break;
case ID_CodecPrivate:
entry.codecPrivate = readBlob(elem);
break;
case ID_Audio:
case ID_Video:
entry.bMetadata = readBlob(elem);
break;
case ID_DefaultDuration:
entry.defaultDuration = readNumber(elem);
break;
case ID_FlagLacing:
drop = readNumber(elem) != lacingExpected;
break;
default:
System.out.println();
break;
}
ensure(elem);
}
if (!drop) {
trackEntries.add(entry);
}
ensure(elem_trackEntry);
}
WebMTrack[] entries = new WebMTrack[trackEntries.size()];
trackEntries.toArray(entries);
for (WebMTrack entry : entries) {
switch (entry.trackType) {
case 1:
entry.kind = TrackKind.Video;
break;
case 2:
entry.kind = TrackKind.Audio;
break;
default:
entry.kind = TrackKind.Other;
break;
}
}
return entries;
}
private SimpleBlock readSimpleBlock(Element ref) throws IOException {
SimpleBlock obj = new SimpleBlock(ref);
obj.dataSize = stream.position();
obj.trackNumber = readEncodedNumber();
obj.relativeTimeCode = stream.readShort();
obj.flags = (byte) stream.read();
obj.dataSize = (ref.offset + ref.size) - stream.position();
if (obj.dataSize < 0) {
throw new IOException(String.format("Unexpected SimpleBlock element size, missing %s bytes", -obj.dataSize));
}
return obj;
}
private Cluster readCluster(Element ref) throws IOException {
Cluster obj = new Cluster(ref);
Element elem = untilElement(ref, ID_Timecode);
if (elem == null) {
throw new NoSuchElementException("Cluster at " + String.valueOf(ref.offset) + " without Timecode element");
}
obj.timecode = readNumber(elem);
return obj;
}
//</editor-fold>
//<editor-fold defaultstate="collapsed" desc="class helpers">
class Element {
int type;
long offset;
long contentSize;
long size;
}
public class Info {
public long timecodeScale;
public long duration;
}
public class WebMTrack {
public long trackNumber;
protected int trackType;
public String codecId;
public byte[] codecPrivate;
public byte[] bMetadata;
public TrackKind kind;
public long defaultDuration;
}
public class Segment {
Segment(Element ref) {
this.ref = ref;
this.firstClusterInSegment = true;
}
public Info info;
WebMTrack[] tracks;
private Element currentCluster;
private final Element ref;
boolean firstClusterInSegment;
public Cluster getNextCluster() throws IOException {
if (done) {
return null;
}
if (firstClusterInSegment && segment.currentCluster != null) {
firstClusterInSegment = false;
return readCluster(segment.currentCluster);
}
ensure(segment.currentCluster);
Element elem = untilElement(segment.ref, ID_Cluster);
if (elem == null) {
return null;
}
segment.currentCluster = elem;
return readCluster(segment.currentCluster);
}
}
public class SimpleBlock {
public TrackDataChunk data;
SimpleBlock(Element ref) {
this.ref = ref;
}
public long trackNumber;
public short relativeTimeCode;
public byte flags;
public long dataSize;
private final Element ref;
public boolean isKeyframe() {
return (flags & 0x80) == 0x80;
}
}
public class Cluster {
Element ref;
SimpleBlock currentSimpleBlock = null;
public long timecode;
Cluster(Element ref) {
this.ref = ref;
}
boolean check() {
return stream.position() >= (ref.offset + ref.size);
}
public SimpleBlock getNextSimpleBlock() throws IOException {
if (check()) {
return null;
}
if (currentSimpleBlock != null) {
ensure(currentSimpleBlock.ref);
}
while (!check()) {
Element elem = untilElement(ref, ID_SimpleBlock);
if (elem == null) {
return null;
}
currentSimpleBlock = readSimpleBlock(elem);
if (currentSimpleBlock.trackNumber == tracks[selectedTrack].trackNumber) {
currentSimpleBlock.data = new TrackDataChunk(stream, (int) currentSimpleBlock.dataSize);
return currentSimpleBlock;
}
ensure(elem);
}
return null;
}
}
//</editor-fold>
}

728
app/src/main/java/org/schabi/newpipe/streams/WebMWriter.java Normal file
View file

@ -0,0 +1,728 @@
package org.schabi.newpipe.streams;
import org.schabi.newpipe.streams.WebMReader.Cluster;
import org.schabi.newpipe.streams.WebMReader.Segment;
import org.schabi.newpipe.streams.WebMReader.SimpleBlock;
import org.schabi.newpipe.streams.WebMReader.WebMTrack;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import org.schabi.newpipe.streams.io.SharpStream;
/**
*
* @author kapodamy
*/
public class WebMWriter {
private final static int BUFFER_SIZE = 8 * 1024;
private final static int DEFAULT_TIMECODE_SCALE = 1000000;
private final static int INTERV = 100;// 100ms on 1000000us timecode scale
private final static int DEFAULT_CUES_EACH_MS = 5000;// 100ms on 1000000us timecode scale
private WebMReader.WebMTrack[] infoTracks;
private SharpStream[] sourceTracks;
private WebMReader[] readers;
private boolean done = false;
private boolean parsed = false;
private long written = 0;
private Segment[] readersSegment;
private Cluster[] readersCluter;
private int[] predefinedDurations;
private byte[] outBuffer;
public WebMWriter(SharpStream... source) {
sourceTracks = source;
readers = new WebMReader[sourceTracks.length];
infoTracks = new WebMTrack[sourceTracks.length];
outBuffer = new byte[BUFFER_SIZE];
}
public WebMTrack[] getTracksFromSource(int sourceIndex) throws IllegalStateException {
if (done) {
throw new IllegalStateException("already done");
}
if (!parsed) {
throw new IllegalStateException("All sources must be parsed first");
}
return readers[sourceIndex].getAvailableTracks();
}
public void parseSources() throws IOException, IllegalStateException {
if (done) {
throw new IllegalStateException("already done");
}
if (parsed) {
throw new IllegalStateException("already parsed");
}
try {
for (int i = 0; i < readers.length; i++) {
readers[i] = new WebMReader(sourceTracks[i]);
readers[i].parse();
}
} finally {
parsed = true;
}
}
public void selectTracks(int... trackIndex) throws IOException {
try {
readersSegment = new Segment[readers.length];
readersCluter = new Cluster[readers.length];
predefinedDurations = new int[readers.length];
for (int i = 0; i < readers.length; i++) {
infoTracks[i] = readers[i].selectTrack(trackIndex[i]);
predefinedDurations[i] = -1;
readersSegment[i] = readers[i].getNextSegment();
}
} finally {
parsed = true;
}
}
public long getBytesWritten() {
return written;
}
public boolean isDone() {
return done;
}
public boolean isParsed() {
return parsed;
}
public void close() {
done = true;
parsed = true;
for (SharpStream src : sourceTracks) {
src.dispose();
}
sourceTracks = null;
readers = null;
infoTracks = null;
readersSegment = null;
readersCluter = null;
outBuffer = null;
}
public void build(SharpStream out) throws IOException, RuntimeException {
if (!out.canRewind()) {
throw new IOException("The output stream must be allow seek");
}
makeEBML(out);
long offsetSegmentSizeSet = written + 5;
long offsetInfoDurationSet = written + 94;
long offsetClusterSet = written + 58;
long offsetCuesSet = written + 75;
ArrayList<byte[]> listBuffer = new ArrayList<>(4);
/* segment */
listBuffer.add(new byte[]{
0x18, 0x53, (byte) 0x80, 0x67, 0x01,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00// segment content size
});
long baseSegmentOffset = written + listBuffer.get(0).length;
/* seek head */
listBuffer.add(new byte[]{
0x11, 0x4d, (byte) 0x9b, 0x74, (byte) 0xbe,
0x4d, (byte) 0xbb, (byte) 0x8b,
0x53, (byte) 0xab, (byte) 0x84, 0x15, 0x49, (byte) 0xa9, 0x66, 0x53,
(byte) 0xac, (byte) 0x81, /*info offset*/ 0x43,
0x4d, (byte) 0xbb, (byte) 0x8b, 0x53, (byte) 0xab,
(byte) 0x84, 0x16, 0x54, (byte) 0xae, 0x6b, 0x53, (byte) 0xac, (byte) 0x81,
/*tracks offset*/ 0x6a,
0x4d, (byte) 0xbb, (byte) 0x8e, 0x53, (byte) 0xab, (byte) 0x84, 0x1f,
0x43, (byte) 0xb6, 0x75, 0x53, (byte) 0xac, (byte) 0x84, /*cluster offset [2]*/ 0x00, 0x00, 0x00, 0x00,
0x4d, (byte) 0xbb, (byte) 0x8e, 0x53, (byte) 0xab, (byte) 0x84, 0x1c, 0x53,
(byte) 0xbb, 0x6b, 0x53, (byte) 0xac, (byte) 0x84, /*cues offset [7]*/ 0x00, 0x00, 0x00, 0x00
});
/* info */
listBuffer.add(new byte[]{
0x15, 0x49, (byte) 0xa9, 0x66, (byte) 0xa2, 0x2a, (byte) 0xd7, (byte) 0xb1
});
listBuffer.add(encode(DEFAULT_TIMECODE_SCALE, true));// this value MUST NOT exceed 4 bytes
listBuffer.add(new byte[]{0x44, (byte) 0x89, (byte) 0x84,
0x00, 0x00, 0x00, 0x00,// info.duration
/* MuxingApp */
0x4d, (byte) 0x80, (byte) 0x87, 0x4E,
0x65, 0x77, 0x50, 0x69, 0x70, 0x65, // "NewPipe" binary string
/* WritingApp */
0x57, 0x41, (byte) 0x87, 0x4E,
0x65, 0x77, 0x50, 0x69, 0x70, 0x65// "NewPipe" binary string
});
/* tracks */
listBuffer.addAll(makeTracks());
for (byte[] buff : listBuffer) {
dump(buff, out);
}
// reserve space for Cues element, but is a waste of space (actually is 64 KiB)
// TODO: better Cue maker
long cueReservedOffset = written;
dump(new byte[]{(byte) 0xec, 0x20, (byte) 0xff, (byte) 0xfb}, out);
int reserved = (1024 * 63) - 4;
while (reserved > 0) {
int write = Math.min(reserved, outBuffer.length);
out.write(outBuffer, 0, write);
reserved -= write;
written += write;
}
// Select a track for the cue
int cuesForTrackId = selectTrackForCue();
long nextCueTime = infoTracks[cuesForTrackId].trackType == 1 ? -1 : 0;
ArrayList<KeyFrame> keyFrames = new ArrayList<>(32);
//ArrayList<Block> chunks = new ArrayList<>(readers.length);
ArrayList<Long> clusterOffsets = new ArrayList<>(32);
ArrayList<Integer> clusterSizes = new ArrayList<>(32);
long duration = 0;
int durationFromTrackId = 0;
byte[] bTimecode = makeTimecode(0);
int firstClusterOffset = (int) written;
long currentClusterOffset = makeCluster(out, bTimecode, 0, clusterOffsets, clusterSizes);
long baseTimecode = 0;
long limitTimecode = -1;
int limitTimecodeByTrackId = cuesForTrackId;
int blockWritten = Integer.MAX_VALUE;
int newClusterByTrackId = -1;
while (blockWritten > 0) {
blockWritten = 0;
int i = 0;
while (i < readers.length) {
Block bloq = getNextBlockFrom(i);
if (bloq == null) {
i++;
continue;
}
if (bloq.data == null) {
blockWritten = 1;// fake block
newClusterByTrackId = i;
i++;
continue;
}
if (newClusterByTrackId == i) {
limitTimecodeByTrackId = i;
newClusterByTrackId = -1;
baseTimecode = bloq.absoluteTimecode;
limitTimecode = baseTimecode + INTERV;
bTimecode = makeTimecode(baseTimecode);
currentClusterOffset = makeCluster(out, bTimecode, currentClusterOffset, clusterOffsets, clusterSizes);
}
if (cuesForTrackId == i) {
if ((nextCueTime > -1 && bloq.absoluteTimecode >= nextCueTime) || (nextCueTime < 0 && bloq.isKeyframe())) {
if (nextCueTime > -1) {
nextCueTime += DEFAULT_CUES_EACH_MS;
}
keyFrames.add(
new KeyFrame(baseSegmentOffset, currentClusterOffset - 7, written, bTimecode.length, bloq.absoluteTimecode)
);
}
}
writeBlock(out, bloq, baseTimecode);
blockWritten++;
if (bloq.absoluteTimecode > duration) {
duration = bloq.absoluteTimecode;
durationFromTrackId = bloq.trackNumber;
}
if (limitTimecode < 0) {
limitTimecode = bloq.absoluteTimecode + INTERV;
continue;
}
if (bloq.absoluteTimecode >= limitTimecode) {
if (limitTimecodeByTrackId != i) {
limitTimecode += INTERV - (bloq.absoluteTimecode - limitTimecode);
}
i++;
}
}
}
makeCluster(out, null, currentClusterOffset, null, clusterSizes);
long segmentSize = written - offsetSegmentSizeSet - 7;
// final step write offsets and sizes
out.rewind();
written = 0;
skipTo(out, offsetSegmentSizeSet);
writeLong(out, segmentSize);
if (predefinedDurations[durationFromTrackId] > -1) {
duration += predefinedDurations[durationFromTrackId];// this value is full-filled in makeTrackEntry() method
}
skipTo(out, offsetInfoDurationSet);
writeFloat(out, duration);
firstClusterOffset -= baseSegmentOffset;
skipTo(out, offsetClusterSet);
writeInt(out, firstClusterOffset);
skipTo(out, cueReservedOffset);
/* Cue */
dump(new byte[]{0x1c, 0x53, (byte) 0xbb, 0x6b, 0x20, 0x00, 0x00}, out);
for (KeyFrame keyFrame : keyFrames) {
for (byte[] buffer : makeCuePoint(cuesForTrackId, keyFrame)) {
dump(buffer, out);
if (written >= (cueReservedOffset + 65535 - 16)) {
throw new IOException("Too many Cues");
}
}
}
short cueSize = (short) (written - cueReservedOffset - 7);
/* EBML Void */
ByteBuffer voidBuffer = ByteBuffer.allocate(4);
voidBuffer.putShort((short) 0xec20);
voidBuffer.putShort((short) (firstClusterOffset - written - 4));
dump(voidBuffer.array(), out);
out.rewind();
written = 0;
skipTo(out, offsetCuesSet);
writeInt(out, (int) (cueReservedOffset - baseSegmentOffset));
skipTo(out, cueReservedOffset + 5);
writeShort(out, cueSize);
for (int i = 0; i < clusterSizes.size(); i++) {
skipTo(out, clusterOffsets.get(i));
byte[] size = ByteBuffer.allocate(4).putInt(clusterSizes.get(i) | 0x200000).array();
out.write(size, 1, 3);
written += 3;
}
}
private Block getNextBlockFrom(int internalTrackId) throws IOException {
if (readersSegment[internalTrackId] == null) {
readersSegment[internalTrackId] = readers[internalTrackId].getNextSegment();
if (readersSegment[internalTrackId] == null) {
return null;// no more blocks in the selected track
}
}
if (readersCluter[internalTrackId] == null) {
readersCluter[internalTrackId] = readersSegment[internalTrackId].getNextCluster();
if (readersCluter[internalTrackId] == null) {
readersSegment[internalTrackId] = null;
return getNextBlockFrom(internalTrackId);
}
}
SimpleBlock res = readersCluter[internalTrackId].getNextSimpleBlock();
if (res == null) {
readersCluter[internalTrackId] = null;
return new Block();// fake block to indicate the end of the cluster
}
Block bloq = new Block();
bloq.data = res.data;
bloq.dataSize = (int) res.dataSize;
bloq.trackNumber = internalTrackId;
bloq.flags = res.flags;
bloq.absoluteTimecode = convertTimecode(res.relativeTimeCode, readersSegment[internalTrackId].info.timecodeScale, DEFAULT_TIMECODE_SCALE);
bloq.absoluteTimecode += readersCluter[internalTrackId].timecode;
return bloq;
}
private short convertTimecode(int time, long oldTimeScale, int newTimeScale) {
return (short) (time * (newTimeScale / oldTimeScale));
}
private void skipTo(SharpStream stream, long absoluteOffset) throws IOException {
absoluteOffset -= written;
written += absoluteOffset;
stream.skip(absoluteOffset);
}
private void writeLong(SharpStream stream, long number) throws IOException {
byte[] buffer = ByteBuffer.allocate(DataReader.LONG_SIZE).putLong(number).array();
stream.write(buffer, 1, buffer.length - 1);
written += buffer.length - 1;
}
private void writeFloat(SharpStream stream, float number) throws IOException {
byte[] buffer = ByteBuffer.allocate(DataReader.FLOAT_SIZE).putFloat(number).array();
dump(buffer, stream);
}
private void writeShort(SharpStream stream, short number) throws IOException {
byte[] buffer = ByteBuffer.allocate(DataReader.SHORT_SIZE).putShort(number).array();
dump(buffer, stream);
}
private void writeInt(SharpStream stream, int number) throws IOException {
byte[] buffer = ByteBuffer.allocate(DataReader.INTEGER_SIZE).putInt(number).array();
dump(buffer, stream);
}
private void writeBlock(SharpStream stream, Block bloq, long clusterTimecode) throws IOException {
long relativeTimeCode = bloq.absoluteTimecode - clusterTimecode;
if (relativeTimeCode < Short.MIN_VALUE || relativeTimeCode > Short.MAX_VALUE) {
throw new IndexOutOfBoundsException("SimpleBlock timecode overflow.");
}
ArrayList<byte[]> listBuffer = new ArrayList<>(5);
listBuffer.add(new byte[]{(byte) 0xa3});
listBuffer.add(null);// block size
listBuffer.add(encode(bloq.trackNumber + 1, false));
listBuffer.add(ByteBuffer.allocate(DataReader.SHORT_SIZE).putShort((short) relativeTimeCode).array());
listBuffer.add(new byte[]{bloq.flags});
int blockSize = bloq.dataSize;
for (int i = 2; i < listBuffer.size(); i++) {
blockSize += listBuffer.get(i).length;
}
listBuffer.set(1, encode(blockSize, false));
for (byte[] buff : listBuffer) {
dump(buff, stream);
}
int read;
while ((read = bloq.data.read(outBuffer)) > 0) {
stream.write(outBuffer, 0, read);
written += read;
}
}
private byte[] makeTimecode(long timecode) {
ByteBuffer buffer = ByteBuffer.allocate(9);
buffer.put((byte) 0xe7);
buffer.put(encode(timecode, true));
byte[] res = new byte[buffer.position()];
System.arraycopy(buffer.array(), 0, res, 0, res.length);
return res;
}
private long makeCluster(SharpStream stream, byte[] bTimecode, long startOffset, ArrayList<Long> clusterOffsets, ArrayList<Integer> clusterSizes) throws IOException {
if (startOffset > 0) {
clusterSizes.add((int) (written - startOffset));// size for last offset
}
if (clusterOffsets != null) {
/* cluster */
dump(new byte[]{0x1f, 0x43, (byte) 0xb6, 0x75}, stream);
clusterOffsets.add(written);// warning: max cluster size is 256 MiB
dump(new byte[]{0x20, 0x00, 0x00}, stream);
startOffset = written;// size for the this cluster
dump(bTimecode, stream);
return startOffset;
}
return -1;
}
private void makeEBML(SharpStream stream) throws IOException {
// deafult values
dump(new byte[]{
0x1A, 0x45, (byte) 0xDF, (byte) 0xA3, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x1F, 0x42, (byte) 0x86, (byte) 0x81, 0x01,
0x42, (byte) 0xF7, (byte) 0x81, 0x01, 0x42, (byte) 0xF2, (byte) 0x81, 0x04,
0x42, (byte) 0xF3, (byte) 0x81, 0x08, 0x42, (byte) 0x82, (byte) 0x84, 0x77,
0x65, 0x62, 0x6D, 0x42, (byte) 0x87, (byte) 0x81, 0x02,
0x42, (byte) 0x85, (byte) 0x81, 0x02
}, stream);
}
private ArrayList<byte[]> makeTracks() {
ArrayList<byte[]> buffer = new ArrayList<>(1);
buffer.add(new byte[]{0x16, 0x54, (byte) 0xae, 0x6b});
buffer.add(null);
for (int i = 0; i < infoTracks.length; i++) {
buffer.addAll(makeTrackEntry(i, infoTracks[i]));
}
return lengthFor(buffer);
}
private ArrayList<byte[]> makeTrackEntry(int internalTrackId, WebMTrack track) {
byte[] id = encode(internalTrackId + 1, true);
ArrayList<byte[]> buffer = new ArrayList<>(12);
/* track */
buffer.add(new byte[]{(byte) 0xae});
buffer.add(null);
/* track number */
buffer.add(new byte[]{(byte) 0xd7});
buffer.add(id);
/* track uid */
buffer.add(new byte[]{0x73, (byte) 0xc5});
buffer.add(id);
/* flag lacing */
buffer.add(new byte[]{(byte) 0x9c, (byte) 0x81, 0x00});
/* lang */
buffer.add(new byte[]{0x22, (byte) 0xb5, (byte) 0x9c, (byte) 0x83, 0x75, 0x6e, 0x64});
/* codec id */
buffer.add(new byte[]{(byte) 0x86});
buffer.addAll(encode(track.codecId));
/* type */
buffer.add(new byte[]{(byte) 0x83});
buffer.add(encode(track.trackType, true));
/* default duration */
if (track.defaultDuration != 0) {
predefinedDurations[internalTrackId] = (int) Math.ceil(track.defaultDuration / (float) DEFAULT_TIMECODE_SCALE);
buffer.add(new byte[]{0x23, (byte) 0xe3, (byte) 0x83});
buffer.add(encode(track.defaultDuration, true));
}
/* audio/video */
if ((track.trackType == 1 || track.trackType == 2) && valid(track.bMetadata)) {
buffer.add(new byte[]{(byte) (track.trackType == 1 ? 0xe0 : 0xe1)});
buffer.add(encode(track.bMetadata.length, false));
buffer.add(track.bMetadata);
}
/* codec private*/
if (valid(track.codecPrivate)) {
buffer.add(new byte[]{0x63, (byte) 0xa2});
buffer.add(encode(track.codecPrivate.length, false));
buffer.add(track.codecPrivate);
}
return lengthFor(buffer);
}
private ArrayList<byte[]> makeCuePoint(int internalTrackId, KeyFrame keyFrame) {
ArrayList<byte[]> buffer = new ArrayList<>(5);
/* CuePoint */
buffer.add(new byte[]{(byte) 0xbb});
buffer.add(null);
/* CueTime */
buffer.add(new byte[]{(byte) 0xb3});
buffer.add(encode(keyFrame.atTimecode, true));
/* CueTrackPosition */
buffer.addAll(makeCueTrackPosition(internalTrackId, keyFrame));
return lengthFor(buffer);
}
private ArrayList<byte[]> makeCueTrackPosition(int internalTrackId, KeyFrame keyFrame) {
ArrayList<byte[]> buffer = new ArrayList<>(8);
/* CueTrackPositions */
buffer.add(new byte[]{(byte) 0xb7});
buffer.add(null);
/* CueTrack */
buffer.add(new byte[]{(byte) 0xf7});
buffer.add(encode(internalTrackId + 1, true));
/* CueClusterPosition */
buffer.add(new byte[]{(byte) 0xf1});
buffer.add(encode(keyFrame.atCluster, true));
/* CueRelativePosition */
if (keyFrame.atBlock > 0) {
buffer.add(new byte[]{(byte) 0xf0});
buffer.add(encode(keyFrame.atBlock, true));
}
return lengthFor(buffer);
}
private void dump(byte[] buffer, SharpStream stream) throws IOException {
stream.write(buffer);
written += buffer.length;
}
private ArrayList<byte[]> lengthFor(ArrayList<byte[]> buffer) {
long size = 0;
for (int i = 2; i < buffer.size(); i++) {
size += buffer.get(i).length;
}
buffer.set(1, encode(size, false));
return buffer;
}
private byte[] encode(long number, boolean withLength) {
int length = -1;
for (int i = 1; i <= 7; i++) {
if (number < Math.pow(2, 7 * i)) {
length = i;
break;
}
}
if (length < 1) {
throw new ArithmeticException("Can't encode a number of bigger than 7 bytes");
}
if (number == (Math.pow(2, 7 * length)) - 1) {
length++;
}
int offset = withLength ? 1 : 0;
byte[] buffer = new byte[offset + length];
long marker = (long) Math.floor((length - 1) / 8);
for (int i = length - 1, mul = 1; i >= 0; i--, mul *= 0x100) {
long b = (long) Math.floor(number / mul);
if (!withLength && i == marker) {
b = b | (0x80 >> (length - 1));
}
buffer[offset + i] = (byte) b;
}
if (withLength) {
buffer[0] = (byte) (0x80 | length);
}
return buffer;
}
private ArrayList<byte[]> encode(String value) {
byte[] str;
try {
str = value.getBytes("utf-8");
} catch (UnsupportedEncodingException err) {
str = value.getBytes();
}
ArrayList<byte[]> buffer = new ArrayList<>(2);
buffer.add(encode(str.length, false));
buffer.add(str);
return buffer;
}
private boolean valid(byte[] buffer) {
return buffer != null && buffer.length > 0;
}
private int selectTrackForCue() {
int i = 0;
int videoTracks = 0;
int audioTracks = 0;
for (; i < infoTracks.length; i++) {
switch (infoTracks[i].trackType) {
case 1:
videoTracks++;
break;
case 2:
audioTracks++;
break;
}
}
int kind;
if (audioTracks == infoTracks.length) {
kind = 2;
} else if (videoTracks == infoTracks.length) {
kind = 1;
} else if (videoTracks > 0) {
kind = 1;
} else if (audioTracks > 0) {
kind = 2;
} else {
return 0;
}
// TODO: in the adove code, find and select the shortest track for the desired kind
for (i = 0; i < infoTracks.length; i++) {
if (kind == infoTracks[i].trackType) {
return i;
}
}
return 0;
}
class KeyFrame {
KeyFrame(long segment, long cluster, long block, int bTimecodeLength, long timecode) {
atCluster = cluster - segment;
if ((block - bTimecodeLength) > cluster) {
atBlock = (int) (block - cluster);
}
atTimecode = timecode;
}
long atCluster;
int atBlock;
long atTimecode;
}
class Block {
InputStream data;
int trackNumber;
byte flags;
int dataSize;
long absoluteTimecode;
boolean isKeyframe() {
return (flags & 0x80) == 0x80;
}
@Override
public String toString() {
return String.format("trackNumber=%s isKeyFrame=%S absoluteTimecode=%s", trackNumber, (flags & 0x80) == 0x80, absoluteTimecode);
}
}
}

47
app/src/main/java/org/schabi/newpipe/streams/io/SharpStream.java Normal file
View file

@ -0,0 +1,47 @@
package org.schabi.newpipe.streams.io;
import java.io.IOException;
/**
* based c#
*/
public abstract class SharpStream {
public abstract int read() throws IOException;
public abstract int read(byte buffer[]) throws IOException;
public abstract int read(byte buffer[], int offset, int count) throws IOException;
public abstract long skip(long amount) throws IOException;
public abstract int available();
public abstract void rewind() throws IOException;
public abstract void dispose();
public abstract boolean isDisposed();
public abstract boolean canRewind();
public abstract boolean canRead();
public abstract boolean canWrite();
public abstract void write(byte value) throws IOException;
public abstract void write(byte[] buffer) throws IOException;
public abstract void write(byte[] buffer, int offset, int count) throws IOException;
public abstract void flush() throws IOException;
public void setLength(long length) throws IOException {
throw new IOException("Not implemented");
}
}

4
app/src/main/java/us/shandian/giga/postprocessing/Mp4DashMuxer.java
View file

@ -1,7 +1,7 @@
package us.shandian.giga.postprocessing; package us.shandian.giga.postprocessing;
import org.schabi.newpipe.extractor.utils.Mp4DashWriter; import org.schabi.newpipe.streams.Mp4DashWriter;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;

2
app/src/main/java/us/shandian/giga/postprocessing/Postprocessing.java
View file

@ -2,7 +2,7 @@ package us.shandian.giga.postprocessing;
import android.os.Message; import android.os.Message;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;

2
app/src/main/java/us/shandian/giga/postprocessing/TestAlgo.java
View file

@ -2,7 +2,7 @@ package us.shandian.giga.postprocessing;
import android.util.Log; import android.util.Log;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;
import java.util.Random; import java.util.Random;

45
app/src/main/java/us/shandian/giga/postprocessing/TttmlConverter.java
View file

@ -1,16 +1,25 @@
package us.shandian.giga.postprocessing; package us.shandian.giga.postprocessing;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import android.util.Log;
import org.schabi.newpipe.extractor.utils.SubtitleConverter;
import org.schabi.newpipe.streams.io.SharpStream;
import org.schabi.newpipe.streams.SubtitleConverter;
import org.xml.sax.SAXException;
import java.io.IOException; import java.io.IOException;
import java.text.ParseException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPathExpressionException;
import us.shandian.giga.get.DownloadMission; import us.shandian.giga.get.DownloadMission;
import us.shandian.giga.postprocessing.io.SharpInputStream; import us.shandian.giga.postprocessing.io.SharpInputStream;
/** /**
* @author kapodamy * @author kapodamy
*/ */
class TttmlConverter extends Postprocessing { class TttmlConverter extends Postprocessing {
private static final String TAG = "TttmlConverter";
TttmlConverter(DownloadMission mission) { TttmlConverter(DownloadMission mission) {
super(mission); super(mission);
@ -26,14 +35,32 @@ class TttmlConverter extends Postprocessing {
if (format == null || format.equals("ttml")) { if (format == null || format.equals("ttml")) {
SubtitleConverter ttmlDumper = new SubtitleConverter(); SubtitleConverter ttmlDumper = new SubtitleConverter();
int res = ttmlDumper.dumpTTML( try {
sources[0], ttmlDumper.dumpTTML(
out, sources[0],
getArgumentAt(1, "true").equals("true"), out,
getArgumentAt(2, "true").equals("true") getArgumentAt(1, "true").equals("true"),
); getArgumentAt(2, "true").equals("true")
);
} catch (Exception err) {
Log.e(TAG, "subtitle parse failed", err);
return res == 0 ? OK_RESULT : res; if (err instanceof IOException) {
return 1;
} else if (err instanceof ParseException) {
return 2;
} else if (err instanceof SAXException) {
return 3;
} else if (err instanceof ParserConfigurationException) {
return 4;
} else if (err instanceof XPathExpressionException) {
return 7;
}
return 8;
}
return OK_RESULT;
} else if (format.equals("srt")) { } else if (format.equals("srt")) {
byte[] buffer = new byte[8 * 1024]; byte[] buffer = new byte[8 * 1024];
int read; int read;

8
app/src/main/java/us/shandian/giga/postprocessing/WebMMuxer.java
View file

@ -1,9 +1,9 @@
package us.shandian.giga.postprocessing; package us.shandian.giga.postprocessing;
import org.schabi.newpipe.extractor.utils.WebMReader.TrackKind; import org.schabi.newpipe.streams.WebMReader.TrackKind;
import org.schabi.newpipe.extractor.utils.WebMReader.WebMTrack; import org.schabi.newpipe.streams.WebMReader.WebMTrack;
import org.schabi.newpipe.extractor.utils.WebMWriter; import org.schabi.newpipe.streams.WebMWriter;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;

2
app/src/main/java/us/shandian/giga/postprocessing/io/ChunkFileInputStream.java
View file

@ -1,6 +1,6 @@
package us.shandian.giga.postprocessing.io; package us.shandian.giga.postprocessing.io;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;

2
app/src/main/java/us/shandian/giga/postprocessing/io/CircularFile.java
View file

@ -1,6 +1,6 @@
package us.shandian.giga.postprocessing.io; package us.shandian.giga.postprocessing.io;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;

2
app/src/main/java/us/shandian/giga/postprocessing/io/FileStream.java
View file

@ -1,6 +1,6 @@
package us.shandian.giga.postprocessing.io; package us.shandian.giga.postprocessing.io;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;

2
app/src/main/java/us/shandian/giga/postprocessing/io/SharpInputStream.java
View file

@ -7,7 +7,7 @@ package us.shandian.giga.postprocessing.io;
import android.support.annotation.NonNull; import android.support.annotation.NonNull;
import org.schabi.newpipe.extractor.utils.io.SharpStream; import org.schabi.newpipe.streams.io.SharpStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;