Architecture Implementation of Android Screen Mirroring to Mac/iPhone

┌─ Source Side (Phone) ──────────────────────────────────────────────────────┐
│                                                                             │
│  ┌─ Platform Layer (Android) ──────────────────────────────────────────┐    │
│  │  Screen Capture (MediaProjection)                                   │    │
│  │       ↓                                                             │    │
│  │  Hardware Encoder (MediaCodec)                                      │    │
│  │  H.264/H.265 │ 1080p@60fps │ 8Mbps │ IDR Response                  │    │
│  │       ↓                                                             │    │
│  │  Encoded Frame Callback → NAL Fragmentation                         │    │
│  └───────┬────────────────────────────────────────────────────────────┘    │
│          ▼                                                                  │
│  ┌─ Mirroring Transport SDK (Source Mode) ─────────────────────────────┐    │
│  │  AES Encryption → RTP Packetization (Seq# + Timestamp) → UDP/TCP Send  │    │
│  │  ← Receive Sink Feedback: Packet Loss / IDR Request / Bitrate Adjustment │
│  └──────────────────────────────────────────────────────────────────────┘    │
└──────────────────────────────────┬─────────────────────────────────────────┘
                                   │ Wi-Fi LAN
                                   ▼
┌─ Sink Side (Mac / iPhone / Pad) ───────────────────────────────────────────┐
│  ┌─ Mirroring Transport SDK (Sink Mode) ────────────────────────────────┐  │
│  │  UDP/TCP Receive → RTP Reassembly → AES Decrypt → Dispatch by Media Type│
│  │  → Send Feedback to Source: Loss Stats / IDR Request / Suggested Bitrate │
│  └────────────────────────────────┬────────────────┬────────────────────┘  │
│                                   │ Video Callback  │ Audio Callback        │
│                                   ▼                ▼                        │
│  ┌─ Platform Layer (Mac/iPhone respective) ──────────────────────────────┐  │
│  │  VideoDecoder (VideoToolbox) → RenderManager → Metal Rendering       │  │
│  │  AudioDecoder (PCM/AAC) → AudioQueue Playback                        │  │
│  │  Reverse Control: Event Capture → 30Hz Throttle → Command Encode → UDP → Phone Injection│
│  └────────────────────────────────────────────────────────────────────────┘  │
└──────────────────────────────────────────────────────────────────────────────┘

┌─────────────────────────────────────────────────────────────┐
│  Android Platform Layer                                      │
│                                                             │
│  MediaProjection (Screen Capture)                           │
│       ↓ Surface                                             │
│  MediaCodec (Hardware Encoder)                              │
│       │  H.264/H.265  │  Config: Resolution/FPS/Bitrate/Profile │
│       ↓ Encoded Frame Callback                              │
│  NAL Fragmentation Processing                               │
│       ↓                                                     │
├─────────────────────────────────────────────────────────────┤
│  Mirroring Transport SDK                                     │
│                                                             │
│  ┌──────────┐    ┌──────────┐    ┌───────────────────────┐  │
│  │ AES Encrypt│ → │ RTP Packet│ → │ UDP/TCP Send          │  │
│  └──────────┘    └──────────┘    └───────────────────────┘  │
│       ↑                                                     │
│  Bitrate Adjustment / IDR Request (from Sink feedback)      │
└─────────────────────────────────────────────────────────────┘

struct EncoderConfig {
    CodecType codec = CodecType::H265;  // Default H.265, fallback to H.264 on older devices
    int width = 1920;
    int height = 1080;
    int fps = 60;
    int bitrate = 8 * 1000 * 1000;  // 8 Mbps
    int profile;
    int level;
};

Encoder Output NAL Units
       ↓
  NAL Fragmentation and Encapsulation
       ↓
  RTP Packetization (Sequence# + Timestamp)
       ↓
    AES Encryption
       ↓
  ┌────┴────┐
  │ Network │
  └────┬────┘
 Normal ↙     ↘ Severe Packet Loss
UDP Send     TCP Send

class MediaSender {
public:
    void sendEncodedFrame(const uint8_t* data, size_t size, int64_t pts) {
        auto fragments = fragmentNAL(data, size);

        for (auto& frag : fragments) {
            RTPPacket packet;
            packet.sequenceNumber = nextSequenceNum_++;
            packet.timestamp = toRTPTimestamp(pts);
            packet.payload = encrypt(frag.data, frag.size);

            if (useTCP_) {
                sendTCP(packet.data(), packet.size());
            } else {
                sendUDP(packet.data(), packet.size());
            }
        }
    }

    void onNetworkDegraded() {
        useTCP_ = true;
        encoder_.setBitrate(bitrate_ / 2);
    }

private:
    uint16_t nextSequenceNum_{0};
    bool useTCP_{false};
};

┌─ Source Side ────────────────────────────┐  ┌─ Sink Side ──────────────────────────────┐
│                                           │  │                                           │
│  IMiPlayCastMirrorServer (C++ Interface)  │  │  IMiPlayCastMirrorClient (C++ Interface) │
│                                           │  │                                           │
│  • attachSurface(surface)  Bind capture Surface│  • attachSurface(surface)  Bind render Surface│
│  • setAttribute(type, val) Set encoding params│  • setAttribute(type, val) Set decoding params│
│  • registerVideoPlugin(p)  Register encode CB│  • registerVideoPlugin(p)  Register decode CB│
│  • registerAudioPlugin(p)  Register audio CB  │  • registerAudioPlugin(p)  Register audio CB  │
│  • registerStateCallback() Register state CB  │  • registerStateCallback() Register state CB  │
│  • write(type, data, len, pts) Send encoded frame│  • start(uri) / stop()  Control session       │
│  • start(uri) / stop()  Control session      │  • pause(mediaType) / resume()  Media control  │
│                                           │  │                                           │
└───────────────────────────────────────────┘  └───────────────────────────────────────────┘

class MediaPlugin {
public:
    virtual int32_t onInit(MediaAttribute attr) = 0;  // Receive negotiated media parameters
    virtual int32_t onStart() = 0;
    virtual int32_t onStop() = 0;
    virtual int32_t onPause() = 0;
    virtual int32_t onResume() = 0;
    virtual int32_t onChangeMediaAttribute(int32_t type, MediaAttribute attr) = 0;
};

struct MediaAttribute {
    int32_t width;        // Video width
    int32_t height;       // Video height
    int32_t fps;          // Frame rate
    int8_t  format[50];   // Codec format string (video/avc, etc.)
    int32_t profile;      // Profile
    int32_t level;        // Level
    int32_t bitrate;      // Bitrate (bps)
    int32_t channels;     // Audio channel count
    int32_t sampleBits;   // Audio sample bit depth
    int32_t sampleRate;   // Audio sample rate
};

class StateCallback {
public:
    virtual void onStarted(int32_t localPort) = 0;           // Server started successfully
    virtual void onConnected() = 0;                          // Connection established
    virtual void onDisconnected() = 0;                       // Connection disconnected
    virtual void onPlayed(int32_t status) = 0;               // Media playback started
    virtual void onError(int32_t what, int32_t extra) = 0;   // Error
    virtual void onInfo(int32_t what, int64_t extra) = 0;    // Info notification
    virtual int32_t onReceiveData(int32_t mediaType,         // Media data received
                                   int8_t* data, int32_t len, int64_t pts) = 0;
};

Server.start(uri)                                        Client.start(uri)
       │                                                       │
       ▼                                                       ▼
  Connection Established ─── TCP Handshake / RTSP Signaling ───→ Connection Established
       │                                                       │
       ▼                                                       ▼
onStarted(localPort)                                   onConnected()
       │                                                       │
       ▼                                                       ▼
  Encryption Negotiation ──── AES key/iv Exchange ───────────→ Encryption Negotiation
       │                                                       │
       ▼                                                       ▼
registerMediaPlugin()                                 registerMediaPlugin()
       │                                                       │
       ▼                                                       ▼
  Start Data Stream ──────── RTP over UDP ────────────────→ onReceiveData()
       │                                                     → plugin.onInit(attr)
       │                                                     → plugin.onStart()
       ▼                                                       ▼
onPlayed(0)                                           onPlayed(0)

Application Data Layer: Encoded H.264/H.265 video frames / PCM/AAC audio frames
     ↓
RTP Encapsulation Layer: Sequence# + Timestamp + Payload Type ID
     ↓
Custom Transport Layer (MPT): UDP preferred, with TCP fallback and Lyra Channel support
     ↓
Physical Link: Wi-Fi / P2P / Bluetooth

Source setAttribute()                        Sink setAttribute()
  VideoWidth  = 1920                          VideoWidth  = 1920
  VideoFps    = 60                            VideoFps    = 60
  VideoEncType = H265                         VideoEncType = H264  ← Sink prefers H.264
  VideoBitrate = 8M
       │                                        │
       └──────── Negotiation Result ─────────────┘
                     VideoEncType = H264  (intersection, Sink doesn't support H.265)
                     Other parameters take the minimum
                     → Delivered to upper layer via onInit(attr)

┌─ SDK Protocol Layer ───────────────────────────────────────┐
│  UDP/TCP Receive → AES Decrypt → RTP Reassembly → Media Type Dispatch │
└────────────────────────────┬─────────────┬────────────────┘
                             │ Video       │ Audio
┌─ Codec Engine Layer ──────▼─────────────▼────────────────┐
│                                                            │
│  VideoDecoder (VideoToolbox)   AudioDecoder (PCM/AAC)        │
│       ↓                          ↓                        │
│  RenderManager                AudioPlayer                 │
│  (Jitter Buffer/Frame Scheduler) (AudioQueue Ring Buffer) │
│       ↓                                                   │
├───────┼───────────────────────────────────────────────────┤
│       ↓              UI Rendering Layer                    │
│  MetalRenderView（YUV→RGB / Zero-Copy）                    │
└───────────────────────────────────────────────────────────┘

class SessionClient {
public:
    void registerStateCallback(StateCallback* cb);
    void registerVideoPlugin(VideoPlugin* plugin);
    void registerAudioPlugin(AudioPlugin* plugin);

    int start(const char* uri, const SessionConfig& config);
    int stop();
    int pause();
    int resume();

    int setAttribute(AttributeType type, int value);

private:
    void receiveLoop();
    void decryptAndDispatch(const uint8_t* data, size_t size);
};

class VideoDecoder {
public:
    int onInit(MediaAttribute attr);
    int onStart();
    int onStop();

    int write(const uint8_t* data, size_t size, int64_t pts);
    void setFrameCallback(FrameDecodedCallback cb);

private:
    void decodeLoop();

    RenderManager renderManager_;
    std::unique_ptr<NativeVTDecoder> nativeVTDecoder_;
};

class NativeVTDecoder {
public:
    int init(int width, int height, CodecType codec) {
        // Construct CMVideoFormatDescription from SPS/PPS
        CMVideoFormatDescriptionCreateFromH264ParameterSets(
            nullptr, paramCount, paramPointers, paramSizes, 4,
            &formatDescription_);

        // Create VTDecompressionSession
        VTDecompressionOutputCallbackRecord callback{&onFrameDecoded, this};
        return VTDecompressionSessionCreate(
            nullptr, formatDescription_, decoderSpec,
            destImageBufferAttributes, &callback, &session_);
    }

    int decode(const uint8_t* data, size_t size, int64_t pts) {
        // Annex-B startcode → AVCC length-prefix format conversion
        auto avccData = convertAnnexBToAVCC(data, size);

        // Construct CMSampleBuffer and submit for decoding
        CMBlockBufferRef blockBuffer;
        CMBlockBufferCreateWithMemoryBlock(nullptr, avccData.data(),
            avccData.size(), kCFAllocatorNull, nullptr, 0,
            avccData.size(), 0, &blockBuffer);

        CMSampleBufferRef sampleBuffer;
        CMSampleBufferCreate(nullptr, blockBuffer, true, nullptr,
            nullptr, formatDescription_, 1, 1, &timingInfo,
            0, nullptr, &sampleBuffer);

        VTDecompressionSessionDecodeFrame(session_, sampleBuffer,
            kVTDecodeFrame_EnableAsynchronousDecompression, nullptr, nullptr);

        CFRelease(sampleBuffer);
        CFRelease(blockBuffer);
        return 0;
    }

private:
    static void onFrameDecoded(void* refCon, void*,
                                OSStatus status, VTDecodeInfoFlags,
                                CVImageBufferRef imageBuffer,
                                CMTime pts, CMTime) {
        auto* self = static_cast<NativeVTDecoder*>(refCon);
        if (status == noErr && self->frameCallback_) {
            self->frameCallback_(imageBuffer, CMTimeGetSeconds(pts));
        }
    }

    VTDecompressionSessionRef session_{nullptr};
    CMVideoFormatDescriptionRef formatDescription_{nullptr};
    FrameCallback frameCallback_;
};

Decode Thread receiveLoop():
    ├─ put(AVPacket) → Enqueue to decode queue
    ├─ decodeLoop(): Dequeue packets
    │   └─ NativeVTDecoder.decode() → VTDecompressionSessionDecodeFrame()
    └─ onFrameDecoded() callback → RenderManager.addFrame()

class RenderManager {
public:
    void addFrame(int64_t index, int64_t pts);
    int64_t render();  // Returns wait time until next render (μs)
    uint32_t forceRender();
    void updateJitterBuffer(int32_t bufferMs, int64_t currentTime);
    void setRefreshRate(int rate);

    using RenderCallback = std::function<void(int64_t pts, int64_t localTime, bool dropped)>;
    void setRenderCallback(RenderCallback cb);

private:
    struct FrameInfo {
        int64_t index;
        int64_t pts;
        int64_t decodeClock;
        int64_t renderClock;
    };

    std::deque<FrameInfo> pendingFrames_;
    int64_t lastRenderTime_{0};
    int64_t recommendMinus_{0};        // Recommended render lead time
    int64_t vsyncInterval_{16000};     // V-Sync interval (μs, ~60fps)
    int64_t maxRenderDelayUs_{70000};  // Maximum tolerable render delay
    int maxRenderCache_{0};
};

int64_t RenderManager::render() {
    if (pendingFrames_.empty()) return emptyWaitTime_;

    auto& frame = pendingFrames_.front();
    int64_t now = currentTimeMicros();
    int64_t targetTime = frame.pts + recommendMinus_;

    // Queue backlog: drop old frames, keep only the newest
    if (maxRenderCache_ > 0 && pendingFrames_.size() > maxRenderCache_) {
        while (pendingFrames_.size() > 1) {
            pendingFrames_.pop_front();
            dropCount_++;
        }
    }

    // Frame too old: drop
    if (now - targetTime > maxRenderDelayUs_) {
        pendingFrames_.pop_front();
        dropCount_++;
        return 0;  // Check next frame immediately
    }

    // Not yet time to render: wait
    if (now < targetTime) return targetTime - now;

    // Render
    pendingFrames_.pop_front();
    renderCallback_(frame.pts, now, false);
    return vsyncInterval_;
}

class MetalRenderView {
    private var device: MTLDevice!
    private var commandQueue: MTLCommandQueue!
    private var pipelineState: MTLRenderPipelineState!
    private var textureCache: CVMetalTextureCache!
    private var metalLayer: CAMetalLayer!

    func setupMetal() {
        device = MTLCreateSystemDefaultDevice()
        commandQueue = device.makeCommandQueue()

        metalLayer = CAMetalLayer()
        metalLayer.device = device
        metalLayer.pixelFormat = .bgra8Unorm

        // CVMetalTextureCache: key to zero-copy
        CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &textureCache)

        setupRenderPipeline()
    }

    func render(pixelBuffer: CVPixelBuffer) {
        let width = CVPixelBufferGetWidth(pixelBuffer)
        let height = CVPixelBufferGetHeight(pixelBuffer)

        // Y plane texture (zero-copy: direct IOSurface mapping)
        var texY: CVMetalTexture?
        CVMetalTextureCacheCreateTextureFromImage(
            kCFAllocatorDefault, textureCache,
            pixelBuffer, nil, .r8Unorm, width, height, 0, &texY)

        // UV plane texture
        var texUV: CVMetalTexture?
        CVMetalTextureCacheCreateTextureFromImage(
            kCFAllocatorDefault, textureCache,
            pixelBuffer, nil, .rg8Unorm, width / 2, height / 2, 1, &texUV)

        guard let mtlY = CVMetalTextureGetTexture(texY!),
              let mtlUV = CVMetalTextureGetTexture(texUV!) else { return }

        // Submit GPU draw commands
        guard let drawable = metalLayer.nextDrawable(),
              let commandBuffer = commandQueue.makeCommandBuffer() else { return }

        let descriptor = MTLRenderPassDescriptor()
        descriptor.colorAttachments[0].texture = drawable.texture
        descriptor.colorAttachments[0].loadAction = .clear

        let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor)!
        encoder.setRenderPipelineState(pipelineState)
        encoder.setFragmentTexture(mtlY, index: 0)
        encoder.setFragmentTexture(mtlUV, index: 1)
        encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
        encoder.endEncoding()

        commandBuffer.present(drawable)
        commandBuffer.commit()
    }
}

#include <metal_stdlib>
using namespace metal;

struct VertexOut {
    float4 position [[position]];
    float2 texCoord;
};

fragment float4 fragmentShader(VertexOut in [[stage_in]],
                               texture2d<float> texY [[texture(0)]],
                               texture2d<float> texUV [[texture(1)]]) {
    constexpr sampler s(address::clamp_to_edge, filter::linear);

    float y = texY.sample(s, in.texCoord).r;
    float2 uv = texUV.sample(s, in.texCoord).rg;

    // BT.709 color space conversion
    float y_adj = y - 0.0625;
    float u = uv.x - 0.5;
    float v = uv.y - 0.5;

    float r = y_adj + 1.5748 * v;
    float g = y_adj - 0.1873 * u - 0.4681 * v;
    float b = y_adj + 1.8556 * u;

    return float4(r, g, b, 1.0);
}

class AudioDecoder {
public:
    int write(const uint8_t* data, size_t size, int64_t pts);
    int readPCM(uint8_t* buffer, size_t size, int64_t* pts);

private:
    PCMDecoder pcmDecoder_;   // PCM passthrough (default)
    AACDecoder aacDecoder_;   // FFmpeg AAC → PCM (fallback)
};

class AudioPlayer {
    static constexpr int kBufferCount = 5;

    AudioQueueRef queue_{nullptr};
    AudioQueueBufferRef buffers_[kBufferCount];

    static void onBufferRequest(void* userData, AudioQueueRef queue,
                                 AudioQueueBufferRef buffer) {
        auto* player = static_cast<AudioPlayer*>(userData);
        player->fillBuffer(buffer);
    }

    void fillBuffer(AudioQueueBufferRef buffer) {
        size_t available = ringBuffer_.readableSize();
        size_t toRead = std::min(available, buffer->mAudioDataBytesCapacity);
        ringBuffer_.read(buffer->mAudioData, toRead);
        buffer->mAudioDataByteSize = toRead;
        AudioQueueEnqueueBuffer(queue_, buffer, 0, nullptr);
    }
};

    Sink Side (Mac/iPhone)               Network                Source Side (Phone)
┌───────────────────────┐    ┌──────────────┐    ┌─────────────────────┐
│ Mouse/Touch Event Capture │    │              │    │ Command Parsing      │
│       ↓               │    │  Independent │    │    ↓                │
│ Coordinate Normalize  │ ──►│  UDP Channel │──► │ Coordinate Mapping  │
│       ↓               │    │  (HID Cmds)  │    │    ↓                │
│ 30Hz Throttle → Serialize│    │              │    │ Android Input Injection│
└───────────────────────┘    └──────────────┘    └─────────────────────┘

struct ControlCommand {
    enum ActionType: UInt8 {
        case touchDown  = 0
        case touchMove  = 1
        case touchUp    = 2
        case scroll     = 5
        case keyPress   = 6
    }

    let action: ActionType
    let x: Float         // Normalized coordinate (0.0 - 1.0)
    let y: Float
    let pressure: Float
    let timestamp: UInt64
}

extension MirrorPlayView {
    override func mouseDown(with event: NSEvent) {
        let location = convert(event.locationInWindow, from: nil)
        let cmd = ControlCommand(
            action: .touchDown,
            x: Float(location.x / bounds.width),
            y: Float(location.y / bounds.height),
            pressure: Float(event.pressure),
            timestamp: currentTimeMicros()
        )
        throttler.submit(cmd)
    }

    override func mouseDragged(with event: NSEvent) {
        let location = convert(event.locationInWindow, from: nil)
        throttler.submit(ControlCommand(
            action: .touchMove,
            x: Float(location.x / bounds.width),
            y: Float(location.y / bounds.height),
            pressure: Float(event.pressure),
            timestamp: currentTimeMicros()
        ))
    }
}

class ControlChannel {
    UDPSocket socket_;

    int send(const ControlCommand& cmd) {
        uint8_t buffer[64];
        size_t offset = 0;
        buffer[offset++] = static_cast<uint8_t>(cmd.action);
        memcpy(buffer + offset, &cmd.x, 4);         offset += 4;
        memcpy(buffer + offset, &cmd.y, 4);         offset += 4;
        memcpy(buffer + offset, &cmd.pressure, 4);  offset += 4;
        memcpy(buffer + offset, &cmd.timestamp, 8); offset += 8;
        return socket_.send(buffer, offset);
    }
};

void onCommandReceived(const uint8_t* data, size_t size) {
    auto cmd = deserialize(data, size);

    // Normalized coordinates → actual phone pixel coordinates
    int screenX = static_cast<int>(cmd.x * deviceWidth_);
    int screenY = static_cast<int>(cmd.y * deviceHeight_);

    switch (cmd.action) {
    case TOUCH_DOWN: injectTouchEvent(ACTION_DOWN, screenX, screenY); break;
    case TOUCH_MOVE: injectTouchEvent(ACTION_MOVE, screenX, screenY); break;
    case TOUCH_UP:   injectTouchEvent(ACTION_UP, screenX, screenY);   break;
    }
}

class EventThrottler {
    private let interval: TimeInterval = 1.0 / 30.0  // 33ms
    private var lastSendTime: TimeInterval = 0
    private var pendingCommand: ControlCommand?

    func submit(_ command: ControlCommand) {
        let now = CACurrentMediaTime()

        // touchDown / touchUp send immediately, no throttling
        if command.action == .touchDown || command.action == .touchUp {
            send(command)
            lastSendTime = now
            return
        }

        // touchMove is throttled
        if now - lastSendTime >= interval {
            send(command)
            lastSendTime = now
        } else {
            pendingCommand = command  // Keep latest position, send next time
        }
    }
}