Harden RTSP startup recovery and frame settings

This commit is contained in:
seo-amugae committed 2026-10-08 10:41:16 +09:00
1 parent 9513ca6e15
commit 5241194b9a
3 files changed
+187 -9

No files matched your search

+180 -8
View File
@@ -50,6 +50,9 @@ param(
[ValidateRange(1, 10)]
[int]$RtspHealthFailureThreshold = 2,
[ValidateRange(0, 600)]
[int]$RtspHealthStartupGraceSec = 45,
[ValidateSet(0, 1)]
[int]$AutoInstallDependencies = 1,
@@ -714,7 +717,8 @@ function Assert-IniSettings {
"Port",
"HealthCheckIntervalSec",
"HealthTimeoutSec",
"HealthFailureThreshold"
"HealthFailureThreshold",
"StartupGraceSec"
)
Dependencies = @(
"AutoInstall",
@@ -831,6 +835,10 @@ if ($ConfigWasLoaded) {
if ($setting.Found -and -not $ExplicitParameters.ContainsKey("RtspHealthFailureThreshold")) {
$RtspHealthFailureThreshold = [int]$setting.Value
}
$setting = Get-IniSetting $ini "RTSP" "StartupGraceSec"
if ($setting.Found -and -not $ExplicitParameters.ContainsKey("RtspHealthStartupGraceSec")) {
$RtspHealthStartupGraceSec = [int]$setting.Value
}
$setting = Get-IniSetting $ini "Dependencies" "AutoInstall"
if ($setting.Found -and -not $ExplicitParameters.ContainsKey("AutoInstallDependencies")) {
@@ -862,6 +870,39 @@ if ($ConfigWasLoaded) {
}
}
function Assert-IntegerRange {
param(
[string]$Name,
[int]$Value,
[int]$Minimum,
[int]$Maximum
)
if ($Value -lt $Minimum -or $Value -gt $Maximum) {
throw "$Name 값은 $Minimum 이상 $Maximum 이하여야 합니다: $Value"
}
}
# Parameter validation runs before config.ini is loaded. Validate the final
# effective values again so an invalid INI value cannot bypass ValidateRange.
Assert-IntegerRange "General.ChannelCount" $ChannelCount 1 16
Assert-IntegerRange "General.StableTimeSec" $StableTimeSec 1 60
Assert-IntegerRange "General.PollIntervalSec" $PollIntervalSec 1 60
Assert-IntegerRange "Video.FrameRate" $FrameRate 1 240
Assert-IntegerRange "RTSP.Port" $RtspPort 1 65535
Assert-IntegerRange "RTSP.HealthCheckIntervalSec" $RtspHealthCheckIntervalSec 1 3600
Assert-IntegerRange "RTSP.HealthTimeoutSec" $RtspHealthTimeoutSec 1 60
Assert-IntegerRange "RTSP.HealthFailureThreshold" $RtspHealthFailureThreshold 1 10
Assert-IntegerRange "RTSP.StartupGraceSec" $RtspHealthStartupGraceSec 0 600
if ($VideoBitrate -notmatch '^[0-9]+[kKmM]$') {
throw "Video.Bitrate 형식이 잘못되었습니다: $VideoBitrate"
}
if ($AudioBitrate -notmatch '^[0-9]+[kKmM]$') {
throw "Audio.Bitrate 형식이 잘못되었습니다: $AudioBitrate"
}
if ($AudioSignalThreshold -lt 0.0 -or $AudioSignalThreshold -gt 1.0) {
throw "Audio.SignalThreshold 값은 0.0 이상 1.0 이하여야 합니다: $AudioSignalThreshold"
}
if (-not [string]::IsNullOrWhiteSpace($EncoderPriorityCsv)) {
$EncoderPriority = @($EncoderPriorityCsv -split ',' | ForEach-Object { $_.Trim() })
}
@@ -1033,6 +1074,7 @@ function Get-HiddenWorkerArgumentList {
"-RtspHealthCheckIntervalSec", [string]$RtspHealthCheckIntervalSec,
"-RtspHealthTimeoutSec", [string]$RtspHealthTimeoutSec,
"-RtspHealthFailureThreshold", [string]$RtspHealthFailureThreshold,
"-RtspHealthStartupGraceSec", [string]$RtspHealthStartupGraceSec,
"-AutoInstallDependencies", [string]$AutoInstallDependencies,
"-FFmpegDownloadUrl", $FFmpegDownloadUrl,
"-FFmpegSha256Url", $FFmpegSha256Url,
@@ -1123,6 +1165,7 @@ if ($ShowConfiguration) {
RtspHealthCheckIntervalSec = $RtspHealthCheckIntervalSec
RtspHealthTimeoutSec = $RtspHealthTimeoutSec
RtspHealthFailureThreshold = $RtspHealthFailureThreshold
RtspHealthStartupGraceSec = $RtspHealthStartupGraceSec
AutoInstallDependencies = $AutoInstallDependencies
FFmpegDownloadUrl = $FFmpegDownloadUrl
FFmpegSha256Url = $FFmpegSha256Url
@@ -2733,6 +2776,7 @@ $script:StreamProcesses = @{}
$script:AudioProcesses = @{}
$script:LastRtspHealthCheck = [DateTime]::MinValue
$script:RtspHealthFailures = 0
$script:StreamStartedAt = [DateTime]::MinValue
function Test-ProcessRunning {
param($Process)
@@ -2786,6 +2830,79 @@ function Stop-Streams {
$script:AudioProcesses = @{}
$script:LastRtspHealthCheck = [DateTime]::MinValue
$script:RtspHealthFailures = 0
$script:StreamStartedAt = [DateTime]::MinValue
}
function Get-StreamProcessDiagnosticLines {
$lines = New-Object System.Collections.Generic.List[string]
foreach ($group in @(
[PSCustomObject]@{Name="FFmpeg"; Processes=$script:StreamProcesses},
[PSCustomObject]@{Name="Audio"; Processes=$script:AudioProcesses}
)) {
foreach ($entry in $group.Processes.GetEnumerator()) {
$process = $entry.Value
try {
$process.Refresh()
if ($process.HasExited) {
$lines.Add("$($group.Name)/$($entry.Key): PID=$($process.Id), 종료, ExitCode=$($process.ExitCode)")
}
else {
$lines.Add("$($group.Name)/$($entry.Key): PID=$($process.Id), 실행 중")
}
}
catch {
$lines.Add("$($group.Name)/$($entry.Key): 상태 확인 실패 - $($_.Exception.Message)")
}
}
}
return @($lines)
}
function Write-StreamFailureDiagnostics {
param([string]$Reason)
Write-Log "AV 파이프라인 진단: $Reason"
foreach ($line in @(Get-StreamProcessDiagnosticLines)) {
Write-Log "프로세스: $line"
}
foreach ($path in @(
(Join-Path $LogDir "ffmpeg.log"),
(Join-Path $LogDir "audio-loopback.log")
)) {
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { continue }
$tail = @(Get-Content -LiteralPath $path -Tail 8 -ErrorAction SilentlyContinue)
foreach ($line in $tail) {
if (-not [string]::IsNullOrWhiteSpace($line)) {
Write-Log "$([System.IO.Path]::GetFileName($path)): $line"
}
}
}
}
function Stop-OrphanedPipelineProcesses {
$scriptPath = [System.IO.Path]::GetFullPath($PSCommandPath)
$candidates = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue |
Where-Object { $_.Name -in @("ffmpeg.exe", "powershell.exe") })
foreach ($candidate in $candidates) {
$owned = $false
if ($candidate.Name -eq "ffmpeg.exe" -and
-not [string]::IsNullOrWhiteSpace($candidate.ExecutablePath)) {
$owned = [string]::Equals(
[System.IO.Path]::GetFullPath($candidate.ExecutablePath),
[System.IO.Path]::GetFullPath($FFmpegPath),
[System.StringComparison]::OrdinalIgnoreCase
)
}
elseif ($candidate.Name -eq "powershell.exe" -and $candidate.CommandLine) {
$owned = ($candidate.CommandLine.IndexOf(
$scriptPath,
[System.StringComparison]::OrdinalIgnoreCase
) -ge 0 -and $candidate.CommandLine -match '(?i)-AudioWorker(?:\s|$)')
}
if (-not $owned -or $candidate.ProcessId -eq $PID) { continue }
Write-Log "이전 Watchdog의 잔여 프로세스 정리: $($candidate.Name) PID=$($candidate.ProcessId)"
Stop-Process -Id $candidate.ProcessId -Force -ErrorAction SilentlyContinue
}
}
function Get-RotationFilter {
@@ -2988,6 +3105,7 @@ function Start-Streams {
$script:LastRtspHealthCheck = Get-Date
$script:RtspHealthFailures = 0
$script:StreamStartedAt = Get-Date
}
function Test-StreamsRunning {
@@ -3002,6 +3120,18 @@ function Test-StreamsRunning {
return $true
}
function Test-RtspHealthCheckDue {
param([datetime]$Now = (Get-Date))
if (-not (Test-StreamsRunning)) { return $false }
if ($script:StreamStartedAt -eq [DateTime]::MinValue) { return $false }
if (($Now - $script:StreamStartedAt).TotalSeconds -lt $RtspHealthStartupGraceSec) {
return $false
}
return (($Now - $script:LastRtspHealthCheck).TotalSeconds -ge
$RtspHealthCheckIntervalSec)
}
function New-RtspHealthProbeArguments {
param([int]$Channel)
@@ -3217,14 +3347,16 @@ function Invoke-SimulationTests {
Invoke-SimulationCase "RTSP 실제 프레임 검사 설정 파싱" {
$path = New-SimulationIni (
"[RTSP]`r`nPort=8554`r`nHealthCheckIntervalSec=10`r`n" +
"HealthTimeoutSec=5`r`nHealthFailureThreshold=2`r`n"
"HealthTimeoutSec=5`r`nHealthFailureThreshold=2`r`n" +
"StartupGraceSec=45`r`n"
)
try {
$parsed = Read-IniConfiguration $path
Assert-IniSettings $parsed
return ($parsed.RTSP.HealthCheckIntervalSec -eq "10" -and
$parsed.RTSP.HealthTimeoutSec -eq "5" -and
$parsed.RTSP.HealthFailureThreshold -eq "2")
$parsed.RTSP.HealthFailureThreshold -eq "2" -and
$parsed.RTSP.StartupGraceSec -eq "45")
}
finally { [System.IO.File]::Delete($path) }
}
@@ -3538,6 +3670,7 @@ function Invoke-SimulationTests {
$arguments -match '-RtspHealthCheckIntervalSec\s+10' -and
$arguments -match '-RtspHealthTimeoutSec\s+5' -and
$arguments -match '-RtspHealthFailureThreshold\s+2' -and
$arguments -match '-RtspHealthStartupGraceSec\s+45' -and
$arguments -match '-WatchdogWorker')
}
@@ -3637,6 +3770,42 @@ function Invoke-SimulationTests {
$arguments -contains "$RTSPPublishBase/monitor2")
}
Invoke-SimulationCase "RTSP 시작 유예 중에는 프레임 검사를 실행하지 않음" {
$savedAudioEnabled = $script:AudioEnabled
$savedStreamProcesses = $script:StreamProcesses
$savedAudioProcesses = $script:AudioProcesses
$savedStartedAt = $script:StreamStartedAt
$savedLastCheck = $script:LastRtspHealthCheck
try {
$now = Get-Date
$alive = [PSCustomObject]@{HasExited=$false}
$script:AudioEnabled = 0
$script:StreamProcesses = @{pipeline=$alive}
$script:AudioProcesses = @{}
$script:LastRtspHealthCheck = $now.AddHours(-1)
$script:StreamStartedAt = $now
$blockedDuringGrace = -not (Test-RtspHealthCheckDue -Now $now)
$script:StreamStartedAt = $now.AddSeconds(-($RtspHealthStartupGraceSec + 1))
$allowedAfterGrace = Test-RtspHealthCheckDue -Now $now
return ($blockedDuringGrace -and $allowedAfterGrace)
}
finally {
$script:AudioEnabled = $savedAudioEnabled
$script:StreamProcesses = $savedStreamProcesses
$script:AudioProcesses = $savedAudioProcesses
$script:StreamStartedAt = $savedStartedAt
$script:LastRtspHealthCheck = $savedLastCheck
}
}
Invoke-SimulationCase "INI FPS 허용 범위 밖 값 거부" {
try {
Assert-IntegerRange "Video.FrameRate" 0 1 240
return $false
}
catch { return $true }
}
Invoke-SimulationCase "비 HTTPS 의존성 URL 거부" {
try { Assert-HttpsDependencyUrl "http://example.com/file.zip" "test"; return $false }
catch { return $true }
@@ -3751,6 +3920,8 @@ try {
$hasMutex = $mutex.WaitOne(0, $false)
if (-not $hasMutex) { throw "Screen RTSP Watchdog가 이미 실행 중입니다." }
Stop-OrphanedPipelineProcesses
Register-ObjectEvent -InputObject ([Microsoft.Win32.SystemEvents]) `
-EventName PowerModeChanged -SourceIdentifier $powerEventSourceIdentifier | Out-Null
$powerEventRegistered = $true
@@ -3791,10 +3962,11 @@ try {
else { Write-Log "NIC 선택: IPv4 기본 경로 자동 감지" }
if ($HealthTarget) { Write-Log "통신 확인 대상: $HealthTarget" }
Write-Log (
"RTSP 프레임 검사: ${RtspHealthCheckIntervalSec}초 주기 / " +
"${RtspHealthTimeoutSec}초 제한 / " +
"RTSP 프레임 검사: 시작 후 ${RtspHealthStartupGraceSec}초 유예 / " +
"${RtspHealthCheckIntervalSec}초 주기 / ${RtspHealthTimeoutSec}초 제한 / " +
"${RtspHealthFailureThreshold}회 연속 실패 시 재시작"
)
Write-Log "영상 설정: ${FrameRate}fps / 채널당 $VideoBitrate"
$lastSystemSignature = ""
$healthySince = $null
@@ -3905,6 +4077,7 @@ try {
if (($StreamProcesses.Count -gt 0 -or $AudioProcesses.Count -gt 0) -and
-not (Test-StreamsRunning)) {
Write-Log "FFmpeg 또는 보조 프로세스 비정상 종료 감지"
Write-StreamFailureDiagnostics "프로세스 종료 감지"
Stop-Streams
$healthySince = $null
}
@@ -3912,9 +4085,7 @@ try {
# A live FFmpeg PID does not prove that DDA and both RTSP outputs
# are progressing. A new local receiver must decode one frame from
# every channel within the configured deadline.
if ((Test-StreamsRunning) -and
((Get-Date) - $script:LastRtspHealthCheck).TotalSeconds -ge
$RtspHealthCheckIntervalSec) {
if (Test-RtspHealthCheckDue) {
$script:LastRtspHealthCheck = Get-Date
$rtspHealth = Test-RtspStreamFrames
@@ -3934,6 +4105,7 @@ try {
if ($script:RtspHealthFailures -ge $RtspHealthFailureThreshold) {
Write-Log "RTSP 송출 정지 감지 - 전체 AV 파이프라인 재시작"
Write-StreamFailureDiagnostics $rtspHealth.Reason
Stop-Streams
$healthySince = $null
}