Skip to content

Commit 4ac07ca

Browse files
heng30heng30
committed
[+] integrate monitor cursor position to recorder
1 parent f08da85 commit 4ac07ca

File tree

1 file changed

+151
-47
lines changed

1 file changed

+151
-47
lines changed

‎lib/recorder/src/recorder.rs‎

Lines changed: 151 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::{
2-
AudioRecorder, EncodedFrame, Frame, FrameUser, ProgressState, RecorderConfig, RecorderError,
3-
Resolution, SimpleFpsCounter, SpeakerRecorder, StatsUser, VideoEncoder,
4-
platform_speaker_recoder, speaker_recorder::SpeakerRecorderConfig,
2+
AudioRecorder, CursorTracker, CursorTrackerConfig, EncodedFrame, Frame, FrameUser,
3+
ProgressState, RecorderConfig, RecorderError, Resolution, SimpleFpsCounter, SpeakerRecorder,
4+
StatsUser, VideoEncoder, platform_speaker_recoder, speaker_recorder::SpeakerRecorderConfig,
55
};
66
use crossbeam::channel::{Receiver, Sender, bounded};
77
use derive_setters::Setters;
@@ -12,12 +12,16 @@ use mp4m::{
1212
AudioConfig, AudioProcessor, AudioProcessorConfigBuilder, Mp4Processor,
1313
Mp4ProcessorConfigBuilder, OutputDestination, VideoConfig, VideoFrameType,
1414
};
15-
use screen_capture::{Capture, CaptureStreamConfig, MonitorCursorPositionConfig, ScreenCapture};
15+
use once_cell::sync::Lazy;
16+
use screen_capture::{
17+
Capture, CaptureStreamConfig, LogicalSize, MonitorCursorPositionConfig, Position, Rectangle,
18+
ScreenCapture, ScreenInfoError,
19+
};
1620
use spin_sleep::SpinSleeper;
1721
use std::{
1822
path::PathBuf,
1923
sync::{
20-
Arc,
24+
Arc, Mutex,
2125
atomic::{AtomicBool, AtomicU64, Ordering},
2226
},
2327
thread::{self, JoinHandle},
@@ -30,6 +34,8 @@ pub type ResizedImageBuffer = ImageBuffer<Rgb<u8>, Vec<u8>>;
3034
const USER_CHANNEL_SIZE: usize = 64;
3135
const ENCODER_WORKER_CHANNEL_SIZE: usize = 128;
3236
const AUDIO_MIXER_CHANNEL_SIZE: usize = 1024;
37+
const CURSOR_CHANNEL_SIZE: usize = 4094;
38+
static CURSOR_POSITION: Lazy<Mutex<Option<Position>>> = Lazy::new(|| Mutex::new(None));
3339

3440
#[derive(Setters)]
3541
#[setters(prefix = "with_")]
@@ -57,6 +63,7 @@ pub struct RecordingSession {
5763
mp4_writer_worker: Option<JoinHandle<()>>,
5864
h264_frame_sender: Option<Sender<VideoFrameType>>,
5965

66+
crop_region_receiver: Option<Receiver<Rectangle>>,
6067
video_encoder: Option<VideoEncoder>,
6168

6269
// statistic
@@ -92,6 +99,7 @@ impl RecordingSession {
9299
mp4_writer_worker: None,
93100
h264_frame_sender: None,
94101

102+
crop_region_receiver: None,
95103
video_encoder: None,
96104

97105
start_time: std::time::Instant::now(),
@@ -147,7 +155,9 @@ impl RecordingSession {
147155
}
148156

149157
if self.config.enable_cursor_tracking {
150-
self.cursor_thread(screen_capturer.clone())?;
158+
let (crop_region_sender, crop_region_receiver) = bounded(CURSOR_CHANNEL_SIZE);
159+
self.cursor_thread(screen_capturer.clone(), crop_region_sender)?;
160+
self.crop_region_receiver = Some(crop_region_receiver);
151161
}
152162

153163
if let Some(device_name) = self.config.audio_device_name.clone() {
@@ -206,7 +216,7 @@ impl RecordingSession {
206216

207217
pub fn wait(mut self) -> Result<ProgressState, RecorderError> {
208218
let (encoder_sender, encoder_receiver) = bounded(ENCODER_WORKER_CHANNEL_SIZE);
209-
let resize_handle = Self::resize_worker(&self, encoder_sender);
219+
let process_frame_handle = Self::process_frame_worker(&self, encoder_sender);
210220

211221
loop {
212222
match encoder_receiver.recv() {
@@ -256,7 +266,7 @@ impl RecordingSession {
256266
_ => {
257267
log::info!("encoder receiver channel exit...");
258268
self.stop();
259-
self.wait_stop(resize_handle)?;
269+
self.wait_stop(process_frame_handle)?;
260270
break;
261271
}
262272
}
@@ -268,32 +278,67 @@ impl RecordingSession {
268278
fn cursor_thread(
269279
&mut self,
270280
mut screen_capturer: impl ScreenCapture + Clone + Send + 'static,
281+
crop_region_sender: Sender<Rectangle>,
271282
) -> Result<(), RecorderError> {
272283
let stop_sig = self.stop_sig.clone();
273284
let screen_name = self.config.screen_name.clone();
274285

286+
let screen_info = screen_capturer
287+
.available_screens()?
288+
.iter()
289+
.find(|item| item.name == screen_name)
290+
.ok_or(RecorderError::ScreenInfoFailed(ScreenInfoError::Other(
291+
format!("No found screen in cursor monitor thread {screen_name}"),
292+
)))?
293+
.clone();
294+
295+
let target_size = LogicalSize::new(500, 400); // TODO
296+
let (cursor_sender, cursor_receiver) = bounded(CURSOR_CHANNEL_SIZE);
297+
298+
// TODO
299+
let cursor_tracker_config = CursorTrackerConfig::new(
300+
screen_info.logical_size,
301+
target_size,
302+
crop_region_sender,
303+
cursor_receiver,
304+
stop_sig.clone(),
305+
)?
306+
.with_stable_radius(100)
307+
.with_fast_moving_duration(Duration::from_millis(100))
308+
.with_linear_transition_duration(Duration::from_millis(1000))
309+
.with_max_stable_region_duration(Duration::from_secs(3));
310+
275311
thread::spawn(move || {
276-
let screens = match screen_capturer.available_screens() {
312+
let cursor_tracker = match CursorTracker::new(cursor_tracker_config) {
277313
Ok(v) => v,
278314
Err(e) => {
279-
log::warn!("{e}");
315+
log::warn!("New cursor tracker faild: {e}");
280316
return;
281317
}
282318
};
283319

284-
let target_screen = match screens.iter().find(|item| item.name == screen_name) {
285-
Some(v) => v.clone(),
286-
_ => {
287-
log::warn!("No found screen in cursor monitor thread {screen_name}");
288-
return;
289-
}
290-
};
320+
if let Err(e) = cursor_tracker.run() {
321+
log::error!("Run Cursor tracker failed: {e}");
322+
}
291323

292-
let config = MonitorCursorPositionConfig::new(target_screen, stop_sig)
324+
log::info!("Exit cursor tracker thread");
325+
});
326+
327+
let cursor_monitor_stop_sig = stop_sig.clone();
328+
thread::spawn(move || {
329+
{
330+
*CURSOR_POSITION.lock().unwrap() = None;
331+
}
332+
333+
let config = MonitorCursorPositionConfig::new(screen_info, cursor_monitor_stop_sig)
293334
.with_use_transparent_layer_surface(true)
294335
.with_hole_radius(50);
295336

296337
if let Err(e) = screen_capturer.monitor_cursor_position(config, move |position| {
338+
{
339+
*CURSOR_POSITION.lock().unwrap() = Some(Position::new(position.x, position.y));
340+
}
341+
297342
log::info!(
298343
"dimensions: {}x{} at ({}, {}). (x, y) = ({}, {})",
299344
position.output_width,
@@ -303,9 +348,15 @@ impl RecordingSession {
303348
position.x,
304349
position.y
305350
);
351+
352+
if let Err(e) = cursor_sender.try_send((Instant::now(), position)) {
353+
log::warn!("cursor sender failed: {e}");
354+
}
306355
}) {
307-
log::error!("{e}");
356+
log::error!("monitor cursor position faield: {e}");
308357
}
358+
359+
log::info!("Exit monitor cursor position thread");
309360
});
310361

311362
Ok(())
@@ -366,7 +417,7 @@ impl RecordingSession {
366417
Ok(())
367418
}
368419

369-
fn resize_worker(
420+
fn process_frame_worker(
370421
session: &RecordingSession,
371422
encoder_sender: Sender<EncoderChannelData>,
372423
) -> JoinHandle<()> {
@@ -376,6 +427,7 @@ impl RecordingSession {
376427
let frame_sender_user = session.frame_sender_user.clone();
377428
let loss_frame_count = session.loss_frame_count.clone();
378429
let total_frame_count = session.total_frame_count.clone();
430+
let enable_cursor_tracking = session.config.enable_cursor_tracking;
379431
let mut fps_counter = SimpleFpsCounter::new();
380432

381433
let handle = thread::spawn(move || {
@@ -399,33 +451,53 @@ impl RecordingSession {
399451
- capture_receiver.len()
400452
);
401453

402-
let resize_now = Instant::now();
403-
match Self::resize_frame(frame, resolution) {
404-
Ok(img) => {
405-
if let Some(ref sender) = frame_sender_user {
406-
let frame_user = FrameUser {
407-
stats: StatsUser {
408-
fps,
409-
total_frames: total_frame_count,
410-
loss_frames: loss_frame_count.load(Ordering::Relaxed),
411-
},
412-
buffer: img.clone(),
413-
};
414-
if let Err(e) = sender.try_send(frame_user) {
415-
log::warn!(
416-
"try send frame to user frame channel failed: {e}"
417-
);
418-
}
454+
let now = Instant::now();
455+
let img = if enable_cursor_tracking {
456+
match Self::crop_and_resize_frame(frame, resolution) {
457+
Ok(img) => img,
458+
Err(e) => {
459+
log::warn!("crop and resize frame failed: {e}");
460+
continue;
419461
}
420-
421-
if let Err(e) = encoder_sender.try_send((total_frame_count, img)) {
422-
loss_frame_count.fetch_add(1, Ordering::Relaxed);
423-
log::warn!("resize worker try send failed: {e}");
462+
}
463+
} else {
464+
match Self::resize_frame(frame, resolution) {
465+
Ok(img) => img,
466+
Err(e) => {
467+
log::warn!("resize frame failed: {e}");
468+
continue;
424469
}
425470
}
426-
Err(e) => log::warn!("resize frame failed: {e}"),
471+
};
472+
473+
log::debug!(
474+
"{} frame spent: {:.2?}",
475+
if enable_cursor_tracking {
476+
"crop"
477+
} else {
478+
"resize"
479+
},
480+
now.elapsed()
481+
);
482+
483+
if let Some(ref sender) = frame_sender_user {
484+
let frame_user = FrameUser {
485+
stats: StatsUser {
486+
fps,
487+
total_frames: total_frame_count,
488+
loss_frames: loss_frame_count.load(Ordering::Relaxed),
489+
},
490+
buffer: img.clone(),
491+
};
492+
if let Err(e) = sender.try_send(frame_user) {
493+
log::warn!("try send frame to user frame channel failed: {e}");
494+
}
495+
}
496+
497+
if let Err(e) = encoder_sender.try_send((total_frame_count, img)) {
498+
loss_frame_count.fetch_add(1, Ordering::Relaxed);
499+
log::warn!("resize worker try send failed: {e}");
427500
}
428-
log::debug!("resize frame spent: {:.2?}", resize_now.elapsed());
429501
}
430502
_ => {
431503
log::info!("resize forward thread exit");
@@ -553,7 +625,7 @@ impl RecordingSession {
553625
Ok((audio_sender, speak_sender))
554626
}
555627

556-
fn wait_stop(mut self, resize_handle: JoinHandle<()>) -> Result<(), RecorderError> {
628+
fn wait_stop(mut self, process_frame_handle: JoinHandle<()>) -> Result<(), RecorderError> {
557629
if let Some(audio_recorder) = self.audio_recorder.take() {
558630
audio_recorder.stop();
559631
log::info!("audio recorder exit...");
@@ -575,10 +647,10 @@ impl RecordingSession {
575647
}
576648
}
577649

578-
if let Err(e) = resize_handle.join() {
579-
log::warn!("join resize thread failed: {:?}", e);
650+
if let Err(e) = process_frame_handle.join() {
651+
log::warn!("join process frame thread failed: {:?}", e);
580652
} else {
581-
log::info!("join resize thread successfully");
653+
log::info!("join process frame thread successfully");
582654
}
583655

584656
match self.video_encoder.take().unwrap().flush() {
@@ -658,6 +730,38 @@ impl RecordingSession {
658730
Ok(())
659731
}
660732

733+
fn crop_and_resize_frame(
734+
frame: Frame,
735+
resolution: Resolution,
736+
) -> Result<ResizedImageBuffer, RecorderError> {
737+
// TODO: 裁剪图片并缩放图片
738+
let img = if matches!(resolution, Resolution::Original(_)) {
739+
let img: ImageBuffer<Rgba<u8>, Vec<u8>> = ImageBuffer::from_raw(
740+
frame.cb_data.data.width,
741+
frame.cb_data.data.height,
742+
frame.cb_data.data.pixel_data,
743+
)
744+
.ok_or_else(|| {
745+
RecorderError::ImageProcessingFailed("Failed to create image buffer".to_string())
746+
})?;
747+
748+
let img: ImageBuffer<Rgb<u8>, Vec<u8>> = img.convert();
749+
img
750+
} else {
751+
let (original_width, original_height) =
752+
(frame.cb_data.data.width, frame.cb_data.data.height);
753+
754+
let img = Self::resize_image(
755+
frame.cb_data.data,
756+
resolution.dimensions(original_width, original_height),
757+
)?;
758+
759+
img
760+
};
761+
762+
Ok(img)
763+
}
764+
661765
fn resize_frame(
662766
frame: Frame,
663767
resolution: Resolution,

0 commit comments

Comments
 (0)
⚡