Recover stalled RTSP streams

screen-rtsp.ps1

@@ -41,6 +41,15 @@ param(
     [ValidateRange(1, 65535)]
     [int]$RtspPort = 8554,
 
+    [ValidateRange(1, 3600)]
+    [int]$RtspHealthCheckIntervalSec = 10,
+
+    [ValidateRange(1, 60)]
+    [int]$RtspHealthTimeoutSec = 5,
+
+    [ValidateRange(1, 10)]
+    [int]$RtspHealthFailureThreshold = 2,
+
     [ValidateSet(0, 1)]
     [int]$AutoInstallDependencies = 1,
 
@@ -701,7 +710,12 @@ function Assert-IniSettings {
         Video = @("FrameRate", "Bitrate", "EncoderPriority")
         Audio = @("Enabled", "MicrophoneEnabled", "Bitrate", "SignalThreshold")
         Network = @("NicDescription", "HealthTarget")
-        RTSP = @("Port")
+        RTSP = @(
+            "Port",
+            "HealthCheckIntervalSec",
+            "HealthTimeoutSec",
+            "HealthFailureThreshold"
+        )
         Dependencies = @(
             "AutoInstall",
             "FFmpegUrl",
@@ -805,6 +819,18 @@ if ($ConfigWasLoaded) {
     if ($setting.Found -and -not $ExplicitParameters.ContainsKey("RtspPort")) {
         $RtspPort = [int]$setting.Value
     }
+    $setting = Get-IniSetting $ini "RTSP" "HealthCheckIntervalSec"
+    if ($setting.Found -and -not $ExplicitParameters.ContainsKey("RtspHealthCheckIntervalSec")) {
+        $RtspHealthCheckIntervalSec = [int]$setting.Value
+    }
+    $setting = Get-IniSetting $ini "RTSP" "HealthTimeoutSec"
+    if ($setting.Found -and -not $ExplicitParameters.ContainsKey("RtspHealthTimeoutSec")) {
+        $RtspHealthTimeoutSec = [int]$setting.Value
+    }
+    $setting = Get-IniSetting $ini "RTSP" "HealthFailureThreshold"
+    if ($setting.Found -and -not $ExplicitParameters.ContainsKey("RtspHealthFailureThreshold")) {
+        $RtspHealthFailureThreshold = [int]$setting.Value
+    }
 
     $setting = Get-IniSetting $ini "Dependencies" "AutoInstall"
     if ($setting.Found -and -not $ExplicitParameters.ContainsKey("AutoInstallDependencies")) {
@@ -1004,6 +1030,9 @@ function Get-HiddenWorkerArgumentList {
         "-StableTimeSec", [string]$StableTimeSec,
         "-PollIntervalSec", [string]$PollIntervalSec,
         "-RtspPort", [string]$RtspPort,
+        "-RtspHealthCheckIntervalSec", [string]$RtspHealthCheckIntervalSec,
+        "-RtspHealthTimeoutSec", [string]$RtspHealthTimeoutSec,
+        "-RtspHealthFailureThreshold", [string]$RtspHealthFailureThreshold,
         "-AutoInstallDependencies", [string]$AutoInstallDependencies,
         "-FFmpegDownloadUrl", $FFmpegDownloadUrl,
         "-FFmpegSha256Url", $FFmpegSha256Url,
@@ -1091,6 +1120,9 @@ if ($ShowConfiguration) {
         NicDescription = $NicDescription
         HealthTarget = $HealthTarget
         RtspPort = $RtspPort
+        RtspHealthCheckIntervalSec = $RtspHealthCheckIntervalSec
+        RtspHealthTimeoutSec = $RtspHealthTimeoutSec
+        RtspHealthFailureThreshold = $RtspHealthFailureThreshold
         AutoInstallDependencies = $AutoInstallDependencies
         FFmpegDownloadUrl = $FFmpegDownloadUrl
         FFmpegSha256Url = $FFmpegSha256Url
@@ -2699,6 +2731,8 @@ $script:MediaMTXProcess = $null
 $script:OwnsMediaMTX = $false
 $script:StreamProcesses = @{}
 $script:AudioProcesses = @{}
+$script:LastRtspHealthCheck = [DateTime]::MinValue
+$script:RtspHealthFailures = 0
 
 function Test-ProcessRunning {
     param($Process)
@@ -2750,6 +2784,8 @@ function Stop-Streams {
     }
     $script:StreamProcesses = @{}
     $script:AudioProcesses = @{}
+    $script:LastRtspHealthCheck = [DateTime]::MinValue
+    $script:RtspHealthFailures = 0
 }
 
 function Get-RotationFilter {
@@ -2949,6 +2985,9 @@ function Start-Streams {
         }
         throw "FFmpeg 또는 오디오 loopback worker가 실행 직후 종료되었습니다."
     }
+
+    $script:LastRtspHealthCheck = Get-Date
+    $script:RtspHealthFailures = 0
 }
 
 function Test-StreamsRunning {
@@ -2963,6 +3002,92 @@ function Test-StreamsRunning {
     return $true
 }
 
+function New-RtspHealthProbeArguments {
+    param([int]$Channel)
+
+    return @(
+        "-hide_banner", "-loglevel", "error", "-nostdin",
+        "-rtsp_transport", "tcp",
+        "-rw_timeout", [string]($RtspHealthTimeoutSec * 1000000),
+        "-i", "$RTSPPublishBase/monitor$Channel",
+        "-map", "0:v:0", "-frames:v", "1",
+        "-an", "-sn", "-dn", "-f", "null", "NUL"
+    )
+}
+
+function Test-RtspStreamFrames {
+    $probes = @()
+    $failures = New-Object System.Collections.Generic.List[string]
+
+    try {
+        for ($channel = 1; $channel -le $ChannelCount; $channel++) {
+            $errorLog = Join-Path $LogDir "rtsp-health-monitor$channel.log"
+            Remove-Item -LiteralPath $errorLog -Force -ErrorAction SilentlyContinue
+
+            $process = Start-Process -FilePath $FFmpegPath `
+                -ArgumentList (Join-ProcessArguments (New-RtspHealthProbeArguments $channel)) `
+                -WorkingDirectory $BaseDir -WindowStyle Hidden `
+                -RedirectStandardError $errorLog -PassThru
+
+            $probes += [PSCustomObject]@{
+                Channel = $channel
+                Process = $process
+                ErrorLog = $errorLog
+            }
+        }
+
+        # Start every receiver first and give the group one shared deadline.
+        # This keeps recovery time bounded even when every channel is stalled.
+        $deadline = [DateTime]::UtcNow.AddSeconds($RtspHealthTimeoutSec + 2)
+        foreach ($probe in $probes) {
+            $remainingMs = [Math]::Max(
+                1,
+                [int]($deadline - [DateTime]::UtcNow).TotalMilliseconds
+            )
+            $completed = $probe.Process.WaitForExit($remainingMs)
+
+            if (-not $completed) {
+                Stop-Process -Id $probe.Process.Id -Force -ErrorAction SilentlyContinue
+                $failures.Add(
+                    "monitor$($probe.Channel): ${RtspHealthTimeoutSec}초 내 영상 프레임 없음"
+                )
+                continue
+            }
+
+            $probe.Process.WaitForExit()
+            if ($probe.Process.ExitCode -ne 0) {
+                $detail = "exit=$($probe.Process.ExitCode)"
+                if (Test-Path -LiteralPath $probe.ErrorLog) {
+                    $tail = @(
+                        Get-Content -LiteralPath $probe.ErrorLog -Tail 3 `
+                            -ErrorAction SilentlyContinue
+                    )
+                    if ($tail.Count -gt 0) { $detail = ($tail -join " | ") }
+                }
+                $failures.Add("monitor$($probe.Channel): $detail")
+            }
+        }
+    }
+    catch {
+        $failures.Add($_.Exception.Message)
+    }
+    finally {
+        foreach ($probe in $probes) {
+            try {
+                if (-not $probe.Process.HasExited) {
+                    Stop-Process -Id $probe.Process.Id -Force -ErrorAction SilentlyContinue
+                }
+            }
+            catch { }
+        }
+    }
+
+    return [PSCustomObject]@{
+        Healthy = ($failures.Count -eq 0)
+        Reason = ($failures -join "; ")
+    }
+}
+
 function Get-StreamingReadinessReason {
     param($Screens, $Network, $PlaybackEndpoint, $MicrophoneEndpoint)
 
@@ -3072,6 +3197,21 @@ function Invoke-SimulationTests {
             finally { [System.IO.File]::Delete($path) }
         }
 
+        Invoke-SimulationCase "RTSP 실제 프레임 검사 설정 파싱" {
+            $path = New-SimulationIni (
+                "[RTSP]`r`nPort=8554`r`nHealthCheckIntervalSec=10`r`n" +
+                "HealthTimeoutSec=5`r`nHealthFailureThreshold=2`r`n"
+            )
+            try {
+                $parsed = Read-IniConfiguration $path
+                Assert-IniSettings $parsed
+                return ($parsed.RTSP.HealthCheckIntervalSec -eq "10" -and
+                    $parsed.RTSP.HealthTimeoutSec -eq "5" -and
+                    $parsed.RTSP.HealthFailureThreshold -eq "2")
+            }
+            finally { [System.IO.File]::Delete($path) }
+        }
+
         Invoke-SimulationCase "자동 기록 갱신은 사용자 설정과 주석 보존" {
             $path = New-SimulationIni "# 사용자 주석`r`n[General]`r`nChannelCount=7`r`n"
             try {
@@ -3378,6 +3518,9 @@ function Invoke-SimulationTests {
                 $arguments -match '-AudioEnabled\s+[01]' -and
                 $arguments -match '-MicrophoneEnabled\s+[01]' -and
                 $arguments -match '-RtspPort\s+8554' -and
+                $arguments -match '-RtspHealthCheckIntervalSec\s+10' -and
+                $arguments -match '-RtspHealthTimeoutSec\s+5' -and
+                $arguments -match '-RtspHealthFailureThreshold\s+2' -and
                 $arguments -match '-WatchdogWorker')
         }
 
@@ -3466,6 +3609,15 @@ function Invoke-SimulationTests {
             return (Test-StreamsRunning)
         }
 
+        Invoke-SimulationCase "RTSP 상태 검사는 새 수신기로 실제 영상 1프레임 요구" {
+            $arguments = @(New-RtspHealthProbeArguments 2)
+            return ($arguments -contains '-rw_timeout' -and
+                $arguments -contains '-frames:v' -and
+                $arguments -contains '1' -and
+                $arguments -contains '0:v:0' -and
+                $arguments -contains "$RTSPPublishBase/monitor2")
+        }
+
         Invoke-SimulationCase "비 HTTPS 의존성 URL 거부" {
             try { Assert-HttpsDependencyUrl "http://example.com/file.zip" "test"; return $false }
             catch { return $true }
@@ -3619,6 +3771,11 @@ try {
     if ($NicDescription) { Write-Log "고정 NIC: $NicDescription" }
     else { Write-Log "NIC 선택: IPv4 기본 경로 자동 감지" }
     if ($HealthTarget) { Write-Log "통신 확인 대상: $HealthTarget" }
+    Write-Log (
+        "RTSP 프레임 검사: ${RtspHealthCheckIntervalSec}초 주기 / " +
+        "${RtspHealthTimeoutSec}초 제한 / " +
+        "${RtspHealthFailureThreshold}회 연속 실패 시 재시작"
+    )
 
     $lastSystemSignature = ""
     $healthySince = $null
@@ -3733,6 +3890,37 @@ try {
                 $healthySince = $null
             }
 
+            # A live FFmpeg PID does not prove that DDA and both RTSP outputs
+            # are progressing.  A new local receiver must decode one frame from
+            # every channel within the configured deadline.
+            if ((Test-StreamsRunning) -and
+                ((Get-Date) - $script:LastRtspHealthCheck).TotalSeconds -ge
+                    $RtspHealthCheckIntervalSec) {
+                $script:LastRtspHealthCheck = Get-Date
+                $rtspHealth = Test-RtspStreamFrames
+
+                if ($rtspHealth.Healthy) {
+                    if ($script:RtspHealthFailures -gt 0) {
+                        Write-Log "RTSP 실제 프레임 수신 상태 복구"
+                    }
+                    $script:RtspHealthFailures = 0
+                }
+                else {
+                    $script:RtspHealthFailures++
+                    Write-Log (
+                        "RTSP 실제 프레임 검사 실패 " +
+                        "($($script:RtspHealthFailures)/$RtspHealthFailureThreshold): " +
+                        $rtspHealth.Reason
+                    )
+
+                    if ($script:RtspHealthFailures -ge $RtspHealthFailureThreshold) {
+                        Write-Log "RTSP 송출 정지 감지 - 전체 AV 파이프라인 재시작"
+                        Stop-Streams
+                        $healthySince = $null
+                    }
+                }
+            }
+
             if (-not (Test-StreamsRunning)) {
                 if (-not $healthySince) {
                     $healthySince = Get-Date