Try forward client microphone without encoding

This commit is contained in:
Cabbache 2025-10-19 18:58:39 +02:00
parent 72d1aedcaa
commit 4e2f53cc2f
12 changed files with 355 additions and 7 deletions

View File

@ -25,6 +25,7 @@ src = [
'src/keyboard_sdk.c',
'src/mouse_capture.c',
'src/mouse_sdk.c',
'src/microphone.c',
'src/opengl.c',
'src/options.c',
'src/packet_merger.c',

View File

@ -17,6 +17,7 @@
#include "util/net.h"
#include "util/thread.h"
#include "version.h"
#include "microphone.h"
#ifdef _WIN32
#include <windows.h>
@ -74,11 +75,16 @@ main_scrcpy(int argc, char *argv[]) {
av_register_all();
#endif
/*
#ifdef HAVE_V4L2
if (args.opts.v4l2_device) {
avdevice_register_all();
}
#endif
*/
//needed for capturing microphone
avdevice_register_all();
if (!net_init()) {
ret = SCRCPY_EXIT_FAILURE;

74
app/src/microphone.c Normal file
View File

@ -0,0 +1,74 @@
#include "microphone.h"
#include <libavcodec/avcodec.h>
#include <libavdevice/avdevice.h>
#include <libavformat/avformat.h>
#include <stdio.h>
#include "util/log.h"
#include "util/net.h"
int read_mic(void *data) {
sc_socket mic_socket = *(sc_socket*)data;
const char *input_format_name = "alsa";
const char *device_name = "default";
AVInputFormat *input_format = av_find_input_format(input_format_name);
if (!input_format) {
fprintf(stderr, "Could not find ALSA input format\n");
return 1;
}
AVFormatContext *fmt_ctx = NULL;
if (avformat_open_input(&fmt_ctx, device_name, input_format, NULL) < 0) {
fprintf(stderr, "Could not open ALSA device '%s'\n", device_name);
return 1;
}
if (avformat_find_stream_info(fmt_ctx, NULL) < 0) {
fprintf(stderr, "Could not read stream info\n");
avformat_close_input(&fmt_ctx);
return 1;
}
// ALSA has only one audio stream
int audio_stream_index = 0;
AVCodecParameters *codecpar = fmt_ctx->streams[audio_stream_index]->codecpar;
AVCodec *codec = avcodec_find_decoder(codecpar->codec_id);
if (!codec) {
fprintf(stderr, "Codec not found\n");
avformat_close_input(&fmt_ctx);
return 1;
}
AVCodecContext *codec_ctx = avcodec_alloc_context3(codec);
avcodec_parameters_to_context(codec_ctx, codecpar);
if (avcodec_open2(codec_ctx, codec, NULL) < 0) {
fprintf(stderr, "Could not open codec\n");
avcodec_free_context(&codec_ctx);
avformat_close_input(&fmt_ctx);
return 1;
}
LOGD("sample_fmt = %d\n", codec_ctx->sample_fmt);
LOGD("sample rate = %d\n", codec_ctx->sample_rate);
AVPacket pkt;
AVFrame *frame = av_frame_alloc();
printf("Recording audio (Ctrl+C to stop)...\n");
while (av_read_frame(fmt_ctx, &pkt) >= 0) {
// ALSA always gives audio packets, so skip the stream_index check
if (avcodec_send_packet(codec_ctx, &pkt) >= 0) {
while (avcodec_receive_frame(codec_ctx, frame) >= 0) {
int bytes_per_sample = av_get_bytes_per_sample(codec_ctx->sample_fmt);
int data_size = frame->nb_samples * frame->ch_layout.nb_channels * bytes_per_sample;
net_send_all(mic_socket, frame->data[0], data_size);
}
}
av_packet_unref(&pkt);
}
av_frame_free(&frame);
avcodec_free_context(&codec_ctx);
avformat_close_input(&fmt_ctx);
return 0;
}

2
app/src/microphone.h Normal file
View File

@ -0,0 +1,2 @@
#include "util/net.h"
int read_mic(void *);

View File

@ -102,6 +102,7 @@ const struct scrcpy_options scrcpy_options_default = {
.power_on = true,
.video = true,
.audio = true,
.microphone = true,
.require_audio = false,
.kill_adb_on_close = false,
.camera_high_speed = false,

View File

@ -312,6 +312,7 @@ struct scrcpy_options {
bool power_on;
bool video;
bool audio;
bool microphone;
bool require_audio;
bool kill_adb_on_close;
bool camera_high_speed;

View File

@ -26,6 +26,7 @@
#include "recorder.h"
#include "screen.h"
#include "server.h"
#include "microphone.h"
#include "uhid/gamepad_uhid.h"
#include "uhid/keyboard_uhid.h"
#include "uhid/mouse_uhid.h"
@ -453,6 +454,7 @@ scrcpy(struct scrcpy_options *options) {
.display_ime_policy = options->display_ime_policy,
.video = options->video,
.audio = options->audio,
.microphone = options->microphone,
.audio_dup = options->audio_dup,
.show_touches = options->show_touches,
.stay_awake = options->stay_awake,
@ -893,6 +895,14 @@ aoa_complete:
audio_demuxer_started = true;
}
if (options->microphone) {
sc_thread mic_thread;
bool ok = sc_thread_create(&mic_thread, read_mic, "scrcpy-mic", &s->server.mic_socket);
if (!ok) {
goto end;
}
}
// If the device screen is to be turned off, send the control message after
// everything is set up
if (options->control && options->turn_screen_off) {

View File

@ -11,6 +11,7 @@
#include "util/env.h"
#include "util/file.h"
#include "util/log.h"
#include "util/net.h"
#include "util/net_intr.h"
#include "util/process.h"
#include "util/str.h"
@ -562,6 +563,7 @@ sc_server_init(struct sc_server *server, const struct sc_server_params *params,
server->video_socket = SC_SOCKET_NONE;
server->audio_socket = SC_SOCKET_NONE;
server->control_socket = SC_SOCKET_NONE;
server->mic_socket = SC_SOCKET_NONE;
sc_adb_tunnel_init(&server->tunnel);
@ -604,10 +606,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
bool video = server->params.video;
bool audio = server->params.audio;
bool control = server->params.control;
bool microphone = server->params.microphone;
sc_socket video_socket = SC_SOCKET_NONE;
sc_socket audio_socket = SC_SOCKET_NONE;
sc_socket control_socket = SC_SOCKET_NONE;
sc_socket mic_socket = SC_SOCKET_NONE;
if (!tunnel->forward) {
if (video) {
video_socket =
@ -632,6 +636,14 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
goto fail;
}
}
if (microphone) {
mic_socket =
net_accept_intr(&server->intr, tunnel->server_socket);
if (mic_socket == SC_SOCKET_NONE) {
goto fail;
}
}
} else {
uint32_t tunnel_host = server->params.tunnel_host;
if (!tunnel_host) {
@ -688,14 +700,19 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
}
}
// Disable Nagle's algorithm for the control socket
// (it only impacts the sending side, so it is useless to set it
// for the other sockets)
if (control_socket != SC_SOCKET_NONE) {
// Disable Nagle's algorithm for the control socket
// (it only impacts the sending side, so it is useless to set it
// for the other sockets)
bool ok = net_set_tcp_nodelay(control_socket, true);
(void) ok; // error already logged
}
if (mic_socket != SC_SOCKET_NONE) {
bool ok = net_set_tcp_nodelay(mic_socket, true);
(void) ok; // error already logged
}
// we don't need the adb tunnel anymore
sc_adb_tunnel_close(tunnel, &server->intr, serial,
server->device_socket_name);
@ -713,10 +730,12 @@ sc_server_connect_to(struct sc_server *server, struct sc_server_info *info) {
assert(!video || video_socket != SC_SOCKET_NONE);
assert(!audio || audio_socket != SC_SOCKET_NONE);
assert(!control || control_socket != SC_SOCKET_NONE);
assert(!microphone || mic_socket != SC_SOCKET_NONE);
server->video_socket = video_socket;
server->audio_socket = audio_socket;
server->control_socket = control_socket;
server->mic_socket = mic_socket;
return true;
@ -739,6 +758,12 @@ fail:
}
}
if (mic_socket != SC_SOCKET_NONE) {
if (!net_close(mic_socket)) {
LOGW("Could not close microphone socket");
}
}
if (tunnel->enabled) {
// Always leave this function with tunnel disabled
sc_adb_tunnel_close(tunnel, &server->intr, serial,
@ -1128,6 +1153,11 @@ run_server(void *data) {
net_interrupt(server->control_socket);
}
if (server->mic_socket != SC_SOCKET_NONE) {
// There is no control_socket if --no-microphone is set
net_interrupt(server->mic_socket);
}
// Give some delay for the server to terminate properly
#define WATCHDOG_DELAY SC_TICK_FROM_SEC(1)
sc_tick deadline = sc_tick_now() + WATCHDOG_DELAY;
@ -1196,6 +1226,9 @@ sc_server_destroy(struct sc_server *server) {
if (server->control_socket != SC_SOCKET_NONE) {
net_close(server->control_socket);
}
if (server->mic_socket != SC_SOCKET_NONE) {
net_close(server->mic_socket);
}
free(server->serial);
free(server->device_socket_name);

View File

@ -54,6 +54,7 @@ struct sc_server_params {
enum sc_display_ime_policy display_ime_policy;
bool video;
bool audio;
bool microphone;
bool audio_dup;
bool show_touches;
bool stay_awake;
@ -93,6 +94,7 @@ struct sc_server {
sc_socket video_socket;
sc_socket audio_socket;
sc_socket mic_socket;
sc_socket control_socket;
const struct sc_server_callbacks *cbs;

View File

@ -26,6 +26,7 @@ public class Options {
private int scid = -1; // 31-bit non-negative value, or -1
private boolean video = true;
private boolean audio = true;
private boolean microphone = true;
private int maxSize;
private VideoCodec videoCodec = VideoCodec.H264;
private AudioCodec audioCodec = AudioCodec.OPUS;
@ -98,6 +99,10 @@ public class Options {
return audio;
}
public boolean getMicrophone() {
return microphone;
}
public int getMaxSize() {
return maxSize;
}
@ -337,6 +342,9 @@ public class Options {
case "audio":
options.audio = Boolean.parseBoolean(value);
break;
case "microphone":
options.microphone = Boolean.parseBoolean(value);
break;
case "video_codec":
VideoCodec videoCodec = VideoCodec.findByName(value);
if (videoCodec == null) {

View File

@ -1,5 +1,16 @@
package com.genymobile.scrcpy;
import java.util.Objects;
import android.content.Context;
import android.media.AudioAttributes;
import android.media.AudioFormat;
import android.media.AudioManager;
import android.media.AudioTrack;
import android.media.MediaRecorder;
import android.net.LocalSocket;
import java.io.InputStream;
import java.io.BufferedInputStream;
import com.genymobile.scrcpy.audio.AudioCapture;
import com.genymobile.scrcpy.audio.AudioCodec;
import com.genymobile.scrcpy.audio.AudioDirectCapture;
@ -29,9 +40,12 @@ import android.os.Build;
import android.os.Looper;
import android.system.Os;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.List;
@ -68,6 +82,154 @@ public final class Server {
// not instantiable
}
private static AudioAttributes createAudioAttributes(int capturePreset) throws Exception {
AudioAttributes.Builder audioAttributesBuilder = new AudioAttributes.Builder();
Method setCapturePresetMethod =
audioAttributesBuilder.getClass().getDeclaredMethod("setCapturePreset", int.class);
setCapturePresetMethod.invoke(audioAttributesBuilder, capturePreset);
return audioAttributesBuilder.build();
}
//https://github.com/Genymobile/scrcpy/issues/3880#issuecomment-1595722119
public static void poc(Object systemMain, BufferedInputStream bis) throws Exception {
Objects.requireNonNull(systemMain);
// var systemContext = systemMain.getSystemContext();
Method getSystemContextMethod = systemMain.getClass().getDeclaredMethod("getSystemContext");
Context systemContext = (Context) getSystemContextMethod.invoke(systemMain);
Objects.requireNonNull(systemContext);
// var audioMixRuleBuilder = new AudioMixingRule.Builder();
@SuppressLint("PrivateApi")
Class<?> audioMixRuleBuilderClass =
Class.forName("android.media.audiopolicy.AudioMixingRule$Builder");
Object audioMixRuleBuilder = audioMixRuleBuilderClass.newInstance();
try {
// Added in Android 13, but previous versions don't work because lack of permission.
// audioMixRuleBuilder.setTargetMixRole(MIX_ROLE_INJECTOR);
Method setTargetMixRoleMethod =
audioMixRuleBuilder.getClass().getDeclaredMethod("setTargetMixRole", int.class);
int MIX_ROLE_INJECTOR = 1;
setTargetMixRoleMethod.invoke(audioMixRuleBuilder, MIX_ROLE_INJECTOR);
} catch (Exception ignored) {
}
Method addMixRuleMethod = audioMixRuleBuilder.getClass()
.getDeclaredMethod("addMixRule", int.class, Object.class);
int RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET = 0x1 << 1;
// audioMixRuleBuilder.addMixRuleMethod(RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET,
// createAudioAttributes(MediaRecorder.AudioSource.DEFAULT));
addMixRuleMethod.invoke(audioMixRuleBuilder, RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET,
createAudioAttributes(MediaRecorder.AudioSource.DEFAULT));
// audioMixRuleBuilder.addMixRuleMethod(RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET,
// createAudioAttributes(MediaRecorder.AudioSource.MIC));
addMixRuleMethod.invoke(audioMixRuleBuilder, RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET,
createAudioAttributes(MediaRecorder.AudioSource.MIC));
// audioMixRuleBuilder.addMixRuleMethod(RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET,
// createAudioAttributes(MediaRecorder.AudioSource.VOICE_COMMUNICATION));
addMixRuleMethod.invoke(audioMixRuleBuilder, RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET,
createAudioAttributes(
MediaRecorder.AudioSource.VOICE_COMMUNICATION));
// audioMixRuleBuilder.addMixRuleMethod(RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET,
// createAudioAttributes(MediaRecorder.AudioSource.UNPROCESSED));
addMixRuleMethod.invoke(audioMixRuleBuilder, RULE_MATCH_ATTRIBUTE_CAPTURE_PRESET,
createAudioAttributes(MediaRecorder.AudioSource.UNPROCESSED));
// var audioMixingRule = audioMixRuleBuilder.build();
Method audioMixRuleBuildMethod = audioMixRuleBuilder.getClass().getDeclaredMethod("build");
Object audioMixingRule = audioMixRuleBuildMethod.invoke(audioMixRuleBuilder);
Objects.requireNonNull(audioMixingRule);
// var audioMixBuilder = new AudioMix.Builder(audioMixingRule);
@SuppressLint("PrivateApi")
Class<?> audioMixBuilderClass = Class.forName("android.media.audiopolicy.AudioMix$Builder");
Constructor audioMixBuilderConstructor =
audioMixBuilderClass.getDeclaredConstructor(audioMixingRule.getClass());
Object audioMixBuilder = audioMixBuilderConstructor.newInstance(audioMixingRule);
Object audioFormat = new AudioFormat.Builder().setEncoding(AudioFormat.ENCODING_PCM_16BIT)
.setSampleRate(48000)
.setChannelMask(AudioFormat.CHANNEL_IN_STEREO)
.build();
// audioMixBuilder.setFormat(audioFormat);
Method setFormatMethod =
audioMixBuilder.getClass().getDeclaredMethod("setFormat", AudioFormat.class);
setFormatMethod.invoke(audioMixBuilder, audioFormat);
// audioMixBuilder.setRouteFlags(ROUTE_FLAG_LOOP_BACK);
Method setRouteFlagsMethod =
audioMixBuilder.getClass().getDeclaredMethod("setRouteFlags", int.class);
int ROUTE_FLAG_LOOP_BACK = 0x1 << 1;
setRouteFlagsMethod.invoke(audioMixBuilder, ROUTE_FLAG_LOOP_BACK);
// var audioMix = audioMixBuilder.build();
Method audioMixBuildMethod = audioMixBuilder.getClass().getDeclaredMethod("build");
Object audioMix = audioMixBuildMethod.invoke(audioMixBuilder);
Objects.requireNonNull(audioMix);
// var audioPolicyBuilder = new AudioPolicy.Builder(systemContext);
@SuppressLint("PrivateApi")
Class<?> audioPolicyBuilderClass =
Class.forName("android.media.audiopolicy.AudioPolicy$Builder");
Constructor audioPolicyBuilderConstructor =
audioPolicyBuilderClass.getDeclaredConstructor(Context.class);
Object audioPolicyBuilder = audioPolicyBuilderConstructor.newInstance(systemContext);
// audioPolicyBuilder.addMix(audioMix);
Method addMixMethod =
audioPolicyBuilder.getClass().getDeclaredMethod("addMix", audioMix.getClass());
addMixMethod.invoke(audioPolicyBuilder, audioMix);
// var audioPolicy = audioPolicyBuilder.build();
Method audioPolicyBuildMethod = audioPolicyBuilder.getClass().getDeclaredMethod("build");
Object audioPolicy = audioPolicyBuildMethod.invoke(audioPolicyBuilder);
Objects.requireNonNull(audioPolicy);
Object audioManager = (AudioManager) systemContext.getSystemService(AudioManager.class);
// audioManager.registerAudioPolicy(audioPolicy);
Method registerAudioPolicyMethod = audioManager.getClass()
.getDeclaredMethod("registerAudioPolicy", audioPolicy.getClass());
// noinspection DataFlowIssue
int result = (int) registerAudioPolicyMethod.invoke(audioManager, audioPolicy);
if (result != 0) {
Ln.d("registerAudioPolicy failed");
return;
}
// var audioTrack = audioPolicy.createAudioTrackSource(audioMix);
Method createAudioTrackSourceMethod = audioPolicy.getClass()
.getDeclaredMethod("createAudioTrackSource", audioMix.getClass());
AudioTrack audioTrack = (AudioTrack) createAudioTrackSourceMethod.invoke(audioPolicy, audioMix);
Objects.requireNonNull(audioTrack);
audioTrack.play();
/*
byte[] samples = new byte[440 * 100];
for (int i = 0; i < samples.length; i += 1) {
samples[i] = (byte)((i / 40) % 2 == 0 ? 0xff:0x00);
}
*/
new Thread(() -> {
while (true) {
byte[] micData = new byte[50000];
try {
int readlen = bis.read(micData);
int written = audioTrack.write(micData, 0, readlen);
Ln.d("written " + written);
} catch (Exception e) {
Ln.e(e.toString());
}
}
}).start();
}
private static void scrcpy(Options options) throws IOException, ConfigurationException {
if (Build.VERSION.SDK_INT < AndroidVersions.API_31_ANDROID_12 && options.getVideoSource() == VideoSource.CAMERA) {
Ln.e("Camera mirroring is not supported before Android 12");
@ -96,13 +258,26 @@ public final class Server {
boolean control = options.getControl();
boolean video = options.getVideo();
boolean audio = options.getAudio();
boolean microphone = options.getMicrophone();
boolean sendDummyByte = options.getSendDummyByte();
Object systemMain = null;
try {
@SuppressLint("PrivateApi")
Class<?> activityThreadClass = Class.forName("android.app.ActivityThread");
@SuppressLint("DiscouragedPrivateApi")
Method systemMainMethod = activityThreadClass.getDeclaredMethod("systemMain");
systemMain = systemMainMethod.invoke(null);
Objects.requireNonNull(systemMain);
} catch (Exception e) {
}
Workarounds.apply();
List<AsyncProcessor> asyncProcessors = new ArrayList<>();
DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, video, audio, control, sendDummyByte);
DesktopConnection connection = DesktopConnection.open(scid, tunnelForward, video, audio, control, microphone, sendDummyByte);
try {
if (options.getSendDeviceMeta()) {
connection.sendDeviceMeta(Device.getDeviceName());
@ -159,6 +334,15 @@ public final class Server {
}
}
if (microphone) {
try {
LocalSocket s = connection.getMicSocket();
InputStream is = s.getInputStream();
BufferedInputStream bis = new BufferedInputStream(is);
poc(systemMain, bis);
} catch (Exception e) {}
}
Completion completion = new Completion(asyncProcessors.size());
for (AsyncProcessor asyncProcessor : asyncProcessors) {
asyncProcessor.start((fatalError) -> {

View File

@ -12,6 +12,7 @@ import java.io.Closeable;
import java.io.FileDescriptor;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import com.genymobile.scrcpy.util.Ln;
public final class DesktopConnection implements Closeable {
@ -28,14 +29,19 @@ public final class DesktopConnection implements Closeable {
private final LocalSocket controlSocket;
private final ControlChannel controlChannel;
private DesktopConnection(LocalSocket videoSocket, LocalSocket audioSocket, LocalSocket controlSocket) throws IOException {
private final LocalSocket micSocket;
//private final FileDescriptor micFd;
private DesktopConnection(LocalSocket videoSocket, LocalSocket audioSocket, LocalSocket controlSocket, LocalSocket micSocket) throws IOException {
this.videoSocket = videoSocket;
this.audioSocket = audioSocket;
this.controlSocket = controlSocket;
this.micSocket = micSocket;
videoFd = videoSocket != null ? videoSocket.getFileDescriptor() : null;
audioFd = audioSocket != null ? audioSocket.getFileDescriptor() : null;
controlChannel = controlSocket != null ? new ControlChannel(controlSocket) : null;
//micFd = micSocket != null ? micSocket.getFileDescriptor() : null;
}
private static LocalSocket connect(String abstractName) throws IOException {
@ -53,13 +59,14 @@ public final class DesktopConnection implements Closeable {
return SOCKET_NAME_PREFIX + String.format("_%08x", scid);
}
public static DesktopConnection open(int scid, boolean tunnelForward, boolean video, boolean audio, boolean control, boolean sendDummyByte)
public static DesktopConnection open(int scid, boolean tunnelForward, boolean video, boolean audio, boolean control, boolean microphone, boolean sendDummyByte)
throws IOException {
String socketName = getSocketName(scid);
LocalSocket videoSocket = null;
LocalSocket audioSocket = null;
LocalSocket controlSocket = null;
LocalSocket micSocket = null;
try {
if (tunnelForward) {
try (LocalServerSocket localServerSocket = new LocalServerSocket(socketName)) {
@ -87,6 +94,14 @@ public final class DesktopConnection implements Closeable {
sendDummyByte = false;
}
}
if (microphone) {
micSocket = localServerSocket.accept();
if (sendDummyByte) {
// send one byte so the client may read() to detect a connection error
micSocket.getOutputStream().write(0);
sendDummyByte = false;
}
}
}
} else {
if (video) {
@ -98,6 +113,9 @@ public final class DesktopConnection implements Closeable {
if (control) {
controlSocket = connect(socketName);
}
if (microphone) {
micSocket = connect(socketName);
}
}
} catch (IOException | RuntimeException e) {
if (videoSocket != null) {
@ -109,10 +127,13 @@ public final class DesktopConnection implements Closeable {
if (controlSocket != null) {
controlSocket.close();
}
if (micSocket != null) {
micSocket.close();
}
throw e;
}
return new DesktopConnection(videoSocket, audioSocket, controlSocket);
return new DesktopConnection(videoSocket, audioSocket, controlSocket, micSocket);
}
private LocalSocket getFirstSocket() {
@ -172,6 +193,11 @@ public final class DesktopConnection implements Closeable {
return audioFd;
}
public LocalSocket getMicSocket()
{
return micSocket;
}
public ControlChannel getControlChannel() {
return controlChannel;
}