package com.cagneymoreau.videoencoding; import android.util.Log; import com.cagneymoreau.network.RobotPoint; import com.cagneymoreau.sensors.MyVideo; import com.cagneymoreau.utility.Debug; import java.io.IOException; import java.io.InputStream; import java.net.DatagramPacket; import java.net.DatagramSocket; import java.net.InetAddress; import java.net.Socket; import java.util.Random; public class TransferH264 extends Thread { private final static String TAG = "TRANSFERH264"; long packetCounter = 0; int nextCountDue = 0; //Network Connection Stuff InputStream in; //This input stream is from the camera device RobotPoint robotPoint; //Here is our object controlling our network connections MyVideo myVideo; // this is where Im getting my sps and pps info from DatagramSocket rtpSocket; //This is the socket we send from private InetAddress destAddress; //receiving ip private int destPort; //receiving port DatagramPacket rtpPacket; boolean notEOF = true; private static final int MTU = 1300; // max packet size /packets could probably go up to 1500 but lets wait here private static final int MTU_EXTRA = 28; // IP & ICMP Headers byte[] test = new byte[8]; //region-------------h.264 NALU FIELDS /* __ Forbidden Zero bit | __ Nal Ref Id | | __ Type | | | +---------------+ |0|1|2|3|4|5|6|7| +-+-+-+-+-+-+-+-+ |F|NRI| Type | +---------------+ -Forbidden zero bit always empty -NRI will be 11 for spspps or 10 for IDR type 5 nalu = IDR type 1 nalu = non IDR type 7 sps type 8 pps these translate int hex values of 65, 41, 67, 68 respectively */ //description stuff byte[] sps = null; byte[] pps = null; byte[] pref = null; byte[] description; // video data stuff byte[] naluHeader = new byte[5]; //nalue header data 4 byte length an 1 byte type byte[] naluBuffer; // this is the buffer which holds the nalu raw payload int type; //header[4] type as an integer int naluLength; // nalubuffers total length as specified by header[0]-[3] as an integer //endregion //region----------------------RTP FIELDS /* https://tools.ietf.org/html/rfc3550 -> RTP Guidelines <- probably not needed https://tools.ietf.org/html/rfc6184 -> H.264 / RTP Guidelines section 5.1 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |V=2|P|X| CC |M| PT | sequence number | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | timestamp | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | synchronization source (SSRC) identifier | +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ | contributing source (CSRC) identifiers | | .... | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 0 1 2 3 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | RTP Header | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |STAP-A NAL HDR | NALU 1 Size | NALU 1 HDR | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | NALU 1 Data | : : + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | | NALU 2 Size | NALU 2 HDR | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | NALU 2 Data | : : | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | :...OPTIONAL RTP padding | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ */ //Header------ byte[] rtpHeader; private final int MAX_SIZE; private static final int HEADER_SIZE = 12; //don't change private static final int VERSION = 2; private static final int PADDING = 0; private static final int EXTENSION = 0; private static final int CSRC_COUNT = 0; private static final int MARKER = 0; //change for each TransferH264 object private final int SSRC ; //change or can change frequently private int marker; private int payloadType; private int sequenceNumber; private int timeStamp; private int payloadSize; private byte[] payload; private long start = 0, duration; //endregion //region--------------------------------------------------------------set up and run method { MAX_SIZE = MTU - MTU_EXTRA; // Note sure these are needed for udp but probably not significant // TODO: 9/17/2018 req? //SSRC = new Random().nextInt(); //identifies to the reciever that the packet belongs to a certain sender //had to create this before starting stream //tightly coupled and sucks sequenceNumber = new Random().nextInt(1000); //rfc 3500 5.1 suggest random start to protect from attacks https://tools.ietf.org/html/rfc3550 timeStamp = new Random().nextInt(1000); // this is a random to start according to rfc and is incremented by media frame rate or time calcs?? } public TransferH264(InputStream in, Socket useless, RobotPoint robotPoint, MyVideo myVideo, int ssrc) { this.in = in; this.robotPoint = robotPoint; try{ //this socket must be odd and increment from other rtpSocket = new DatagramSocket(6101); // TODO: 9/26/2018 this should increment down from our other udp socket }catch (IOException ioe){ Log.e(TAG, "TransferThread: ", ioe); } SSRC = ssrc; destAddress = robotPoint.getClientTCPAddress(); destPort = robotPoint.getMediaPort(); Log.d(TAG, "TransferH264: address=" + destAddress.toString() + " port= " + String.valueOf(destPort)); byte[][] token = myVideo.getSpspps(); sps = token[0]; pps = token[1]; pref = token[2]; Debug.debugHex("---------", sps, 20); Debug.debugHex("---------", pps, 20); Debug.debugHex("---------", pref, 20); } @Override public void run() { /* try{ streamTCP(); }catch (IOException ioe){ Log.e(TAG, "run: ", ioe ); } */ confirmUDPWorks(); try { // find the mdat box? //build our description buildSPSPPS(); //find th first nalu syncWithNalu(); while (notEOF) { duration = System.nanoTime() - start; buildNalu(); start = System.nanoTime(); readNextHeader(); } in.close(); rtpSocket.close(); } catch (IOException e) { Log.e(getClass().getSimpleName(), "Exception transferring file", e); } } //endregion //here we are testing to see if i can work it with a tcp connection private void streamTCP() throws IOException { buildSPSPPS(); int loopcount = 0; int due = 0; byte[] incoming = new byte[1024]; while(in.read(incoming,0, 1024) > 0){ robotPoint.sendTCPData(incoming); if (due == loopcount){ Log.d(TAG, "streamTCP: description sent"); robotPoint.sendTCPData(description); due+= 150; } loopcount++; } } //region---------------------------------------------------- packetize and format h.264 // get from myvideo / build sps and pps data private void buildSPSPPS() { //without this stream is worthless if (sps == null || pps == null){ notEOF = false; Log.d(TAG, "buildSPSPPS: no sps or pps data"); return; } if (description == null){ description = new byte[sps.length + pps.length + pref.length ]; description[0] = 24; //rtp header trpe 24 = Single-time aggregation packet 5.7.1 // Write NALU 1 size into the array (NALU 1 is the SPS). description[1] = (byte) (sps.length >> 8); description[2] = (byte) (sps.length & 0xFF); // Write NALU 2 size into the array (NALU 2 is the PPS). description[sps.length + 3] = (byte) (pps.length >> 8); description[sps.length + 4] = (byte) (pps.length & 0xFF); //write prefix //System.arraycopy(pref, 0, description, description.length-6, pref.length); // Write NALU 1 into the array, then write NALU 2 into the array. System.arraycopy(sps, 0, description, 3, sps.length); System.arraycopy(pps, 0, description, 5 + sps.length, pps.length); Debug.debugFull(" build spspps ", description); } } // TODO: 9/19/2018 timestamp within should be ignored but verify. obi the time here has no association with any value //called from readNextHeader to send data before each media data private void packetizeDecsription() { buildRTPPacket(24, timeStamp, description, description.length); } // TODO: 9/18/2018 getting random data blobs every 10+/- nalus which are probably chunks?? //scans stream until a header is loaded <- probably expensive private void syncWithNalu() throws IOException { //Log.d(TAG, "syncWithNalu: started - we have no position! invalid data is length = " + String.valueOf(naluLength) + " type: " + String.valueOf(type)); byte save = naluHeader[0]; boolean firstPass = true; int reqLoops = 0; while (true){ reqLoops++; naluHeader[0] = naluHeader[1]; naluHeader[1] = naluHeader[2]; naluHeader[2] = naluHeader[3]; naluHeader[3] = naluHeader[4]; naluHeader[4] = (byte) in.read(); type = naluHeader[4]&0x1F; if (type == 5 || type == 1) { naluLength = (naluHeader[3]&0xFF | (naluHeader[2]&0xFF)<<8 | (naluHeader[1]&0xFF)<<16 | (naluHeader[0]&0xFF)<<24) - 1; //minus type for header!!! if (naluLength > 0 && naluLength < 200000) { //Log.d(TAG, "naluSearch: found length = " + String.valueOf(naluLength) + " of type: " + String.valueOf(type) + " try req: " + String.valueOf(reqLoops)); break; } if (naluLength==0) { Log.d(TAG, "naluSearch: null nalu"); } }else if (firstPass) { firstPass = false; int testtype = (naluHeader[2] &0xFF | (naluHeader[1]&0xFF)<<8 | (naluHeader[0]&0xFF)<<16 | (save &0xFF)<<24) - 1; //minus type for header!!! //DEBUG BAD NALUS HERE String tt = String.valueOf(testtype); byte[] b = new byte[512]; //Debug.debugHex("syncwithnalu " + tt, test, test.length); b[0] = save; b[1] = naluHeader[0]; b[2] = naluHeader[1]; b[3] = naluHeader[2]; b[4] = naluHeader[3]; b[5] = naluHeader[4]; in.read(b, 6, b.length-6); Debug.debugHex("syncwithnalu " , b, 30); } } } //read next header into header fields. expects to be dropped into correct position or it will perform a sync private void readNextHeader() throws IOException { in.read(naluHeader, 0, 5); type = naluHeader[4]&0x1F; // String s1 = String.format("%8s", Integer.toBinaryString(naluHeader[4] & 0xFF)).replace(' ', '0'); // Log.d(TAG, "packetizeNalu: handing type " + s1 + " " + String.valueOf(type)); naluLength = (naluHeader[3]&0xFF | (naluHeader[2]&0xFF)<<8 | (naluHeader[1]&0xFF)<<16 | (naluHeader[0]&0xFF)<<24)- 1; //minus 1 for header!!! if (naluLength >= 200000 || naluLength < 0){ syncWithNalu(); }else{ Log.d(TAG, "readNextHeader success type " + String.valueOf(type) + " length " + String.valueOf(naluLength)); } // IDR is a stand alone picture. sending spspps will ake it readable even in a live stream format without session description protocal if (type == 5){ packetizeDecsription(); } } //build next data which should be video payload private void buildNalu() throws IOException { naluBuffer = new byte[naluLength+5]; //here we recombine our original header to our nalu data to be sent naluBuffer[0] = naluHeader[0]; naluBuffer[1] = naluHeader[1]; naluBuffer[2] = naluHeader[2]; naluBuffer[3] = naluHeader[3]; naluBuffer[4] = naluHeader[4]; in.read(naluBuffer, 5, naluLength); naluLength = naluBuffer.length; test[0] = naluBuffer[naluBuffer.length-4]; test[1] = naluBuffer[naluBuffer.length-3]; test[2] = naluBuffer[naluBuffer.length-2]; test[3] = naluBuffer[naluBuffer.length-1]; test[4] = naluBuffer[naluBuffer.length-4]; test[5] = naluBuffer[naluBuffer.length-3]; test[6] = naluBuffer[naluBuffer.length-2]; test[7] = naluBuffer[naluBuffer.length-1]; timeStampCalulations(); //here we calc the time between reading each nalu. each nalu must have different time stamp // String s1 = String.format("%8s", Integer.toBinaryString(naluHeader[4] & 0xFF)).replace(' ', '0'); //Log.d(TAG, "packetizeNalu: expected raw " + s1); //debugPackets("buildnalu ", naluBuffer); packetizeNalu(); } //called by build nalu to send asap after building it will packetize nalu split it up or whatever // single nalu (section 5.6) or fu-a see -> https://tools.ietf.org/html/rfc6184 section 5.4 private void packetizeNalu() { //Debug.checkNaluDataV2(naluBuffer); //Log.d(TAG, "packetizeNalu: " + String.valueOf(naluBuffer.length)); //error checking int nalLengthChecker = (naluBuffer[3]&0xFF | (naluBuffer[2]&0xFF)<<8 | (naluBuffer[1]&0xFF)<<16 | (naluBuffer[0]&0xFF)<<24); //original nal length nalLengthChecker += 4; //here we add for the header 0-3... remember length includes nal header[4]type already int writtenLength = 0; int bytesAdded = 2; //here is our fua header byte[] buffer; //single nalu if (naluLength+1 <= MAX_SIZE ) { buffer = new byte[naluLength+1]; buffer[0] = naluHeader[4]; System.arraycopy(naluBuffer, 0, buffer, 1, naluLength); buildRTPPacket(type, timeStamp, naluBuffer, naluLength); } // nalu is split like fu-a type else{ /* FU-indicator FU header +---------------+ +---------------+ +---------------+ |0|1|2|3|4|5|6|7| |0|1|2|3|4|5|6|7| +-+-+-+-+-+-+-+-+ +-+-+-+-+-+-+-+-+ FU Payload |F|NRI| TypeofFU| |S|E|R| Type | +---------------+ +---------------+ +---------------+ See rfc 6184 5.8 figure 15-ish See rfc 6184 5.3 "the value of NRI to 11" FU-A is type 28 */ byte[] fuaHeader = new byte[2]; fuaHeader[0] = 0b01111100; //set indicator with "11" and type decimal 28 = FU-A -> 01111100 fuaHeader[1] = (byte) (naluHeader[4] & 0x1F); //set header 3-7 int tally = 0; int tocopy; boolean secondloop = false; while(tally < naluLength) { tocopy = (naluLength - tally); //see whats left to write if (tocopy >= MAX_SIZE-2) // we minus 2 to make space for both header bytes { tocopy = MAX_SIZE-2; //fit into max allowable packet size buffer = new byte[MAX_SIZE]; }else{ buffer = new byte[tocopy+2]; //or shrink buffer to whats left plus header } if (secondloop) //turn ser to double zero on second loop { fuaHeader[1] = (byte) (fuaHeader[1]^(1 << 7)); secondloop = false; //String s1 = String.format("%8s", Integer.toBinaryString(fuaHeader[1] & 0xFF)).replace(' ', '0'); //Log.d(TAG, "packetizeNalu: center header " + s1); } if (tally == 0) //first nalu in multi part. set SER...see above { fuaHeader[1] += 0x80; secondloop = true; //String s1 = String.format("%8s", Integer.toBinaryString(fuaHeader[1] & 0xFF)).replace(' ', '0'); //Log.d(TAG, "packetizeNalu: adjusted header " + s1); } System.arraycopy(naluBuffer, tally, buffer, 2, tocopy); //copy to buffer skipping first 2 bytes tally += tocopy; if (tally >= naluLength) //weve copied all the data, set ser to last on multipart { fuaHeader[1] += 0x40; //String s1 = String.format("%8s", Integer.toBinaryString(fuaHeader[1] & 0xFF)).replace(' ', '0'); //Log.d(TAG, "packetizeNalu: re-adjut header " + s1); } buffer[0] = fuaHeader[0]; buffer[1] = fuaHeader[1]; buildRTPPacket(fuaHeader[0], timeStamp, buffer, buffer.length); writtenLength += (buffer.length - bytesAdded); //here we count how many bytes were sent to packetizer to compare to our starting amount } if (writtenLength != nalLengthChecker){ Log.e(TAG, "packetizeNalu: Mismatched Size orig: " + String.valueOf(nalLengthChecker) + " written " + String.valueOf(writtenLength), null ); } } //Debug.checkNaluData(naluBuffer); } //endregion //region--------------------------------------------------------------RTP and Network Connection // TODO: 9/17/2018 This buffer might be overriden and we need a que to hold these. udpsocket is linear & fifo?! private void send(byte[] buf, boolean debug) { if (debug)debugOutGoingPacket(buf); rtpPacket = new DatagramPacket(buf, buf.length, destAddress, destPort); try{ rtpSocket.send(rtpPacket); }catch (IOException ioe){ Log.e(TAG, "RTP Packet Send: add: " + destAddress.toString() + " port: " + String.valueOf(destPort), ioe); } } public void buildRTPPacket(int payloadType, int timeStamp, byte[] payload, int payloadLength) { //Log.d(TAG, "buildRTPPacket: " + String.valueOf(payloadType)); //this is the actual packet being sent byte[] rtpPacket = new byte[HEADER_SIZE + payloadLength]; sequenceNumber++; //keep our packet stream linear.splt nalus with same timestamp are ordered by this number rtpHeader = new byte[HEADER_SIZE]; rtpHeader[0] = (byte) 0b10000000; //(byte) (VERSION << 6 | PADDING << 5 | EXTENSION << 4 | CSRC_COUNT); rtpHeader[1] = (byte) payloadType; //ignore market bit //The first byte of a NAL unit co-serves as the RTP payload header -> https://tools.ietf.org/html/rfc6184 5.6 rtpHeader[2] = (byte) (sequenceNumber >> 8); //sequence move bits 8-16 right into the 8 bit buffer rtpHeader[3] = (byte) (sequenceNumber & 0xff); //sequence only keep the the last 8 bits by masking rtpHeader[4] = (byte) (timeStamp >> 24); //time stamp rtpHeader[5] = (byte) (timeStamp >> 18); //time stamp rtpHeader[6] = (byte) (timeStamp >> 8); //time stamp rtpHeader[7] = (byte) (timeStamp & 0xFF); //time stamp rtpHeader[8] = (byte) (SSRC >> 24); //ssrc rtpHeader[9] = (byte) (SSRC >> 16); //ssrc rtpHeader[10] = (byte) (SSRC >> 8); //ssrc rtpHeader[11] = (byte) (SSRC & 0xff); //ssrc // here we load the header into the first 12 bytes and the payload after that System.arraycopy(rtpHeader, 0, rtpPacket,0,HEADER_SIZE); System.arraycopy(payload,0,rtpPacket,HEADER_SIZE, rtpPacket.length-HEADER_SIZE); debugPackets(); //send as soon as its built with true or false for console debugging send(rtpPacket, false); } private void debugOutGoingPacket( byte[] buf) { StringBuilder sb = new StringBuilder(); for (byte b : buf) { sb.append(String.format("%02X", b)); //sb.append(" "); } String futureHex = sb.toString(); sb = new StringBuilder(); for (int i = 0; i < futureHex.length(); i+=2) { String str = futureHex.substring(i, i + 2); sb.append((char) Integer.parseInt(str, 16)); } String s = sb.toString(); Log.d(TAG, "debug outgoing " + s); //payload = s.getBytes(); } private void debugPackets() { if (nextCountDue == packetCounter ){ Log.d(TAG, "packets streaming current count = " + String.valueOf(packetCounter) + " client: " + destAddress.toString() + " port: " + String.valueOf(destPort)); nextCountDue += 1500; } packetCounter++; } //endregion //region--------------------------------------------------------------helpers etc //Calced in billionths and convert to 90khz clock. seems terrible... magic number with < 500 nanos error // each nalue rtp tmestamp progress by adding its own duration so if ts = 20k ave eg. 0 -> 20k -> 40k ->60k //this tells the receiever how long to keep image on screen before replacing with next private void timeStampCalulations() { timeStamp += (int) duration / 11111; //divide nanoseconds by 11,111 to equal a 90khz clock rate } private void confirmUDPWorks() { Log.d(TAG, "confirmUDPWorks: "); //byte[] SD = Base64.decode(myVideo.getSessionDescription(), Base64.DEFAULT); byte[] start = new byte[5]; start[0] = 0x73; start[0] = 0x74; start[0] = 0x61; start[0] = 0x72; start[0] = 0x74; send(start, true); } public void setNotEOF(boolean notEOF) { this.notEOF = notEOF; } //endregion }