I can not find any package to stream what the user is doing out of the application with react native.
I am creating my own plugin
The goal is to send a stream from java to javascript to then send it an external socket server via socket.io
This is my java class to ask authorization and stream
import android.hardware.display.DisplayManager;import android.util.DisplayMetrics;import android.app.Activity;import android.content.Intent;import android.media.projection.MediaProjectionManager;import android.media.projection.MediaProjection;import android.media.MediaRecorder;import android.util.Log;import com.facebook.react.bridge.ReactApplicationContext;import com.facebook.react.bridge.ReactContextBaseJavaModule;import com.facebook.react.bridge.ReactMethod;import com.facebook.react.bridge.Promise;import com.facebook.react.bridge.ActivityEventListener;import com.facebook.react.bridge.BaseActivityEventListener;import static android.content.Context.MEDIA_PROJECTION_SERVICE;public class ScreenMirroringModule extends ReactContextBaseJavaModule { private static final int SCREEN_SHARE_REQUEST = 4242; private static final String S_MIRROR_CANCELLED = "S_MIRROR_CANCELLED"; private Promise sMirrorPromise; private MediaProjectionManager mMediaProjectionManager; private MediaStreamer mMediaStreamer = new MediaStreamer(); private final ActivityEventListener mActivityEventListener = new BaseActivityEventListener() { @Override public void onActivityResult(Activity activity, int requestCode, int resultCode, Intent intent) { if (requestCode == SCREEN_SHARE_REQUEST) { if (sMirrorPromise != null) { if (resultCode == Activity.RESULT_CANCELED) { sMirrorPromise.reject(S_MIRROR_CANCELLED, "Screen mirroring was cancelled"); } else if (resultCode == Activity.RESULT_OK) { DisplayMetrics dm = new DisplayMetrics(); activity.getWindowManager().getDefaultDisplay().getMetrics(dm); mMediaStreamer.reset(); mMediaStreamer.setVideoSource(MediaRecorder.VideoSource.SURFACE); mMediaStreamer.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4); mMediaStreamer.setVideoEncoder(MediaRecorder.VideoEncoder.MPEG_4_SP); mMediaStreamer.setVideoSize(dm.widthPixels, dm.heightPixels); mMediaStreamer.setVideoFrameRate(30); StreamTask st = new StreamTask(); st.execute(mMediaStreamer); try { st.get(); } catch (Exception e) { sMirrorPromise.reject(e); } MediaProjection mMediaProjection = mMediaProjectionManager.getMediaProjection(resultCode, intent); try { mMediaProjection.createVirtualDisplay("MainActivity", dm.widthPixels, dm.heightPixels, dm.densityDpi, DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR, mMediaStreamer.getSurface(), null /*Callbacks*/, null/*Handler*/); mMediaStreamer.start(); } catch (Exception e) { //Log.v("ReactNative", Log.getStackTraceString(e)); sMirrorPromise.reject(e); } sMirrorPromise.resolve("Test"); } } sMirrorPromise = null; } } }; public ScreenMirroringModule(ReactApplicationContext reactContext) { super(reactContext); // Add the listener for `onActivityResult` reactContext.addActivityEventListener(mActivityEventListener); } @Override public String getName() { return "ScreenMirroring"; } @ReactMethod public void stream(String ip, int port, Promise promise) { final Activity activity = getCurrentActivity(); // Store the promise to resolve/reject when picker returns data sMirrorPromise = promise; mMediaProjectionManager = (MediaProjectionManager) activity.getSystemService(MEDIA_PROJECTION_SERVICE); Intent intent = mMediaProjectionManager.createScreenCaptureIntent(); activity.startActivityForResult(intent, SCREEN_SHARE_REQUEST); }}This is my class which is transforming the recording into a stream
package com.ijkoareactapp;import android.media.MediaRecorder;import android.net.LocalServerSocket;import android.net.LocalSocket;import android.net.LocalSocketAddress;import android.util.Log;import java.io.IOException;import java.io.InputStream;public class MediaStreamer extends MediaRecorder { private LocalServerSocket localServerSocket = null; private LocalSocket receiver, sender = null; public void prepare() throws IllegalStateException, IOException { receiver = new LocalSocket(); try { localServerSocket = new LocalServerSocket("screen_mirror_socket"); receiver.connect(new LocalSocketAddress("screen_mirror_socket")); receiver.setReceiveBufferSize(4096); receiver.setSendBufferSize(4096); sender = localServerSocket.accept(); sender.setReceiveBufferSize(4096); sender.setSendBufferSize(4096); } catch (Exception e) { throw new IOException("Can't create local socket !"); } setOutputFile(sender.getFileDescriptor()); try { super.prepare(); } catch (Exception e) { Log.v("ReactNative", Log.getStackTraceString(e)); closeSockets(); throw e; } } public InputStream getInputStream() { InputStream out = null; try { out = receiver.getInputStream(); } catch (IOException e) { } return out; } public void stop() { closeSockets(); super.stop(); } private void closeSockets() { if (localServerSocket != null) { try { localServerSocket.close(); sender.close(); receiver.close(); } catch (IOException e) { } localServerSocket = null; sender = null; receiver = null; } }}And this is the class which is running the task in background because I can not start a local socket in main process
package com.ijkoareactapp;import android.os.AsyncTask;import android.util.Log;public class StreamTask extends AsyncTask <MediaStreamer, Integer, Long>{ protected Long doInBackground(MediaStreamer... mediaStreamers) { Log.v("ReactNative", "Start"); for(int i = 0; i < mediaStreamers.length; i++) { try { mediaStreamers[i].prepare(); Log.v("ReactNative", "Prepared"); } catch (Exception e) { Log.v("ReactNative", "Error"); Log.v("ReactNative", Log.getStackTraceString(e)); } } return null; }}The error I have is
08-05 15:28:49.465 555 571 V ReactNative: java.lang.IllegalStateException08-05 15:28:49.465 555 571 V ReactNative: at android.media.MediaRecorder._prepare(Native Method)08-05 15:28:49.465 555 571 V ReactNative: at android.media.MediaRecorder.prepare(MediaRecorder.java:827)08-05 15:28:49.465 555 571 V ReactNative: at com.ijkoareactapp.MediaStreamer.prepare(MediaStreamer.java:37)08-05 15:28:49.465 555 571 V ReactNative: at com.ijkoareactapp.StreamTask.doInBackground(StreamTask.java:14)08-05 15:28:49.465 555 571 V ReactNative: at com.ijkoareactapp.StreamTask.doInBackground(StreamTask.java:6)08-05 15:28:49.465 555 571 V ReactNative: at android.os.AsyncTask$2.call(AsyncTask.java:304)08-05 15:28:49.465 555 571 V ReactNative: at java.util.concurrent.FutureTask.run(FutureTask.java:237)08-05 15:28:49.465 555 571 V ReactNative: at android.os.AsyncTask$SerialExecutor$1.run(AsyncTask.java:243)08-05 15:28:49.465 555 571 V ReactNative: at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1133)08-05 15:28:49.465 555 571 V ReactNative: at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:607)08-05 15:28:49.465 555 571 V ReactNative: at java.lang.Thread.run(Thread.java:761)Reading some documentation it seems that prepare is called before start or after setOutputFormat but this is not the case.
I think that my internal socket connection is not well initialized but I can not find the right way to do it.
As well not sure where should I put the mMediaProjection.createVirtualDisplay, at the moment I have the follwing error should be because the prepare does not works
08-05 15:43:36.077 1575 1575 V ReactNative: java.lang.IllegalStateException: failed to get surface08-05 15:43:36.077 1575 1575 V ReactNative: at android.media.MediaRecorder.getSurface(Native Method)08-05 15:43:36.077 1575 1575 V ReactNative: at com.ijkoareactapp.ScreenMirroringModule$1.onActivityResult(ScreenMirroringModule.java:65)08-05 15:43:36.077 1575 1575 V ReactNative: at com.facebook.react.bridge.ReactContext.onActivityResult(ReactContext.java:262)08-05 15:43:36.077 1575 1575 V ReactNative: at com.facebook.react.ReactInstanceManager.onActivityResult(ReactInstanceManager.java:703)08-05 15:43:36.077 1575 1575 V ReactNative: at com.facebook.react.ReactActivityDelegate.onActivityResult(ReactActivityDelegate.java:124)08-05 15:43:36.077 1575 1575 V ReactNative: at com.facebook.react.ReactActivity.onActivityResult(ReactActivity.java:75)08-05 15:43:36.077 1575 1575 V ReactNative: at android.app.Activity.dispatchActivityResult(Activity.java:6915)08-05 15:43:36.077 1575 1575 V ReactNative: at android.app.ActivityThread.deliverResults(ActivityThread.java:4049)08-05 15:43:36.077 1575 1575 V ReactNative: at android.app.ActivityThread.handleSendResult(ActivityThread.java:4096)08-05 15:43:36.077 1575 1575 V ReactNative: at android.app.ActivityThread.-wrap20(ActivityThread.java)08-05 15:43:36.077 1575 1575 V ReactNative: at android.app.ActivityThread$H.handleMessage(ActivityThread.java:1516)08-05 15:43:36.077 1575 1575 V ReactNative: at android.os.Handler.dispatchMessage(Handler.java:102)08-05 15:43:36.077 1575 1575 V ReactNative: at android.os.Looper.loop(Looper.java:154)08-05 15:43:36.077 1575 1575 V ReactNative: at android.app.ActivityThread.main(ActivityThread.java:6077)08-05 15:43:36.077 1575 1575 V ReactNative: at java.lang.reflect.Method.invoke(Native Method)08-05 15:43:36.077 1575 1575 V ReactNative: at com.android.internal.os.ZygoteInit$MethodAndArgsCaller.run(ZygoteInit.java:866)08-05 15:43:36.077 1575 1575 V ReactNative: at com.android.internal.os.ZygoteInit.main(ZygoteInit.java:756)
