Harden RTSP startup recovery and frame settings
This commit is contained in:
1 parent
9513ca6e15
commit
5241194b9a
3 files changed
+187
-9
No files matched your search
+180
-8
@@ -50,6 +50,9 @@ param(
|
||||
[ValidateRange(1, 10)]
|
||||
[int]$RtspHealthFailureThreshold = 2,
|
||||
|
||||
[ValidateRange(0, 600)]
|
||||
[int]$RtspHealthStartupGraceSec = 45,
|
||||
|
||||
[ValidateSet(0, 1)]
|
||||
[int]$AutoInstallDependencies = 1,
|
||||
|
||||
@@ -714,7 +717,8 @@ function Assert-IniSettings {
|
||||
"Port",
|
||||
"HealthCheckIntervalSec",
|
||||
"HealthTimeoutSec",
|
||||
"HealthFailureThreshold"
|
||||
"HealthFailureThreshold",
|
||||
"StartupGraceSec"
|
||||
)
|
||||
Dependencies = @(
|
||||
"AutoInstall",
|
||||
@@ -831,6 +835,10 @@ if ($ConfigWasLoaded) {
|
||||
if ($setting.Found -and -not $ExplicitParameters.ContainsKey("RtspHealthFailureThreshold")) {
|
||||
$RtspHealthFailureThreshold = [int]$setting.Value
|
||||
}
|
||||
$setting = Get-IniSetting $ini "RTSP" "StartupGraceSec"
|
||||
if ($setting.Found -and -not $ExplicitParameters.ContainsKey("RtspHealthStartupGraceSec")) {
|
||||
$RtspHealthStartupGraceSec = [int]$setting.Value
|
||||
}
|
||||
|
||||
$setting = Get-IniSetting $ini "Dependencies" "AutoInstall"
|
||||
if ($setting.Found -and -not $ExplicitParameters.ContainsKey("AutoInstallDependencies")) {
|
||||
@@ -862,6 +870,39 @@ if ($ConfigWasLoaded) {
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-IntegerRange {
|
||||
param(
|
||||
[string]$Name,
|
||||
[int]$Value,
|
||||
[int]$Minimum,
|
||||
[int]$Maximum
|
||||
)
|
||||
if ($Value -lt $Minimum -or $Value -gt $Maximum) {
|
||||
throw "$Name 값은 $Minimum 이상 $Maximum 이하여야 합니다: $Value"
|
||||
}
|
||||
}
|
||||
|
||||
# Parameter validation runs before config.ini is loaded. Validate the final
|
||||
# effective values again so an invalid INI value cannot bypass ValidateRange.
|
||||
Assert-IntegerRange "General.ChannelCount" $ChannelCount 1 16
|
||||
Assert-IntegerRange "General.StableTimeSec" $StableTimeSec 1 60
|
||||
Assert-IntegerRange "General.PollIntervalSec" $PollIntervalSec 1 60
|
||||
Assert-IntegerRange "Video.FrameRate" $FrameRate 1 240
|
||||
Assert-IntegerRange "RTSP.Port" $RtspPort 1 65535
|
||||
Assert-IntegerRange "RTSP.HealthCheckIntervalSec" $RtspHealthCheckIntervalSec 1 3600
|
||||
Assert-IntegerRange "RTSP.HealthTimeoutSec" $RtspHealthTimeoutSec 1 60
|
||||
Assert-IntegerRange "RTSP.HealthFailureThreshold" $RtspHealthFailureThreshold 1 10
|
||||
Assert-IntegerRange "RTSP.StartupGraceSec" $RtspHealthStartupGraceSec 0 600
|
||||
if ($VideoBitrate -notmatch '^[0-9]+[kKmM]$') {
|
||||
throw "Video.Bitrate 형식이 잘못되었습니다: $VideoBitrate"
|
||||
}
|
||||
if ($AudioBitrate -notmatch '^[0-9]+[kKmM]$') {
|
||||
throw "Audio.Bitrate 형식이 잘못되었습니다: $AudioBitrate"
|
||||
}
|
||||
if ($AudioSignalThreshold -lt 0.0 -or $AudioSignalThreshold -gt 1.0) {
|
||||
throw "Audio.SignalThreshold 값은 0.0 이상 1.0 이하여야 합니다: $AudioSignalThreshold"
|
||||
}
|
||||
|
||||
if (-not [string]::IsNullOrWhiteSpace($EncoderPriorityCsv)) {
|
||||
$EncoderPriority = @($EncoderPriorityCsv -split ',' | ForEach-Object { $_.Trim() })
|
||||
}
|
||||
@@ -1033,6 +1074,7 @@ function Get-HiddenWorkerArgumentList {
|
||||
"-RtspHealthCheckIntervalSec", [string]$RtspHealthCheckIntervalSec,
|
||||
"-RtspHealthTimeoutSec", [string]$RtspHealthTimeoutSec,
|
||||
"-RtspHealthFailureThreshold", [string]$RtspHealthFailureThreshold,
|
||||
"-RtspHealthStartupGraceSec", [string]$RtspHealthStartupGraceSec,
|
||||
"-AutoInstallDependencies", [string]$AutoInstallDependencies,
|
||||
"-FFmpegDownloadUrl", $FFmpegDownloadUrl,
|
||||
"-FFmpegSha256Url", $FFmpegSha256Url,
|
||||
@@ -1123,6 +1165,7 @@ if ($ShowConfiguration) {
|
||||
RtspHealthCheckIntervalSec = $RtspHealthCheckIntervalSec
|
||||
RtspHealthTimeoutSec = $RtspHealthTimeoutSec
|
||||
RtspHealthFailureThreshold = $RtspHealthFailureThreshold
|
||||
RtspHealthStartupGraceSec = $RtspHealthStartupGraceSec
|
||||
AutoInstallDependencies = $AutoInstallDependencies
|
||||
FFmpegDownloadUrl = $FFmpegDownloadUrl
|
||||
FFmpegSha256Url = $FFmpegSha256Url
|
||||
@@ -2733,6 +2776,7 @@ $script:StreamProcesses = @{}
|
||||
$script:AudioProcesses = @{}
|
||||
$script:LastRtspHealthCheck = [DateTime]::MinValue
|
||||
$script:RtspHealthFailures = 0
|
||||
$script:StreamStartedAt = [DateTime]::MinValue
|
||||
|
||||
function Test-ProcessRunning {
|
||||
param($Process)
|
||||
@@ -2786,6 +2830,79 @@ function Stop-Streams {
|
||||
$script:AudioProcesses = @{}
|
||||
$script:LastRtspHealthCheck = [DateTime]::MinValue
|
||||
$script:RtspHealthFailures = 0
|
||||
$script:StreamStartedAt = [DateTime]::MinValue
|
||||
}
|
||||
|
||||
function Get-StreamProcessDiagnosticLines {
|
||||
$lines = New-Object System.Collections.Generic.List[string]
|
||||
foreach ($group in @(
|
||||
[PSCustomObject]@{Name="FFmpeg"; Processes=$script:StreamProcesses},
|
||||
[PSCustomObject]@{Name="Audio"; Processes=$script:AudioProcesses}
|
||||
)) {
|
||||
foreach ($entry in $group.Processes.GetEnumerator()) {
|
||||
$process = $entry.Value
|
||||
try {
|
||||
$process.Refresh()
|
||||
if ($process.HasExited) {
|
||||
$lines.Add("$($group.Name)/$($entry.Key): PID=$($process.Id), 종료, ExitCode=$($process.ExitCode)")
|
||||
}
|
||||
else {
|
||||
$lines.Add("$($group.Name)/$($entry.Key): PID=$($process.Id), 실행 중")
|
||||
}
|
||||
}
|
||||
catch {
|
||||
$lines.Add("$($group.Name)/$($entry.Key): 상태 확인 실패 - $($_.Exception.Message)")
|
||||
}
|
||||
}
|
||||
}
|
||||
return @($lines)
|
||||
}
|
||||
|
||||
function Write-StreamFailureDiagnostics {
|
||||
param([string]$Reason)
|
||||
|
||||
Write-Log "AV 파이프라인 진단: $Reason"
|
||||
foreach ($line in @(Get-StreamProcessDiagnosticLines)) {
|
||||
Write-Log "프로세스: $line"
|
||||
}
|
||||
foreach ($path in @(
|
||||
(Join-Path $LogDir "ffmpeg.log"),
|
||||
(Join-Path $LogDir "audio-loopback.log")
|
||||
)) {
|
||||
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { continue }
|
||||
$tail = @(Get-Content -LiteralPath $path -Tail 8 -ErrorAction SilentlyContinue)
|
||||
foreach ($line in $tail) {
|
||||
if (-not [string]::IsNullOrWhiteSpace($line)) {
|
||||
Write-Log "$([System.IO.Path]::GetFileName($path)): $line"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Stop-OrphanedPipelineProcesses {
|
||||
$scriptPath = [System.IO.Path]::GetFullPath($PSCommandPath)
|
||||
$candidates = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.Name -in @("ffmpeg.exe", "powershell.exe") })
|
||||
foreach ($candidate in $candidates) {
|
||||
$owned = $false
|
||||
if ($candidate.Name -eq "ffmpeg.exe" -and
|
||||
-not [string]::IsNullOrWhiteSpace($candidate.ExecutablePath)) {
|
||||
$owned = [string]::Equals(
|
||||
[System.IO.Path]::GetFullPath($candidate.ExecutablePath),
|
||||
[System.IO.Path]::GetFullPath($FFmpegPath),
|
||||
[System.StringComparison]::OrdinalIgnoreCase
|
||||
)
|
||||
}
|
||||
elseif ($candidate.Name -eq "powershell.exe" -and $candidate.CommandLine) {
|
||||
$owned = ($candidate.CommandLine.IndexOf(
|
||||
$scriptPath,
|
||||
[System.StringComparison]::OrdinalIgnoreCase
|
||||
) -ge 0 -and $candidate.CommandLine -match '(?i)-AudioWorker(?:\s|$)')
|
||||
}
|
||||
if (-not $owned -or $candidate.ProcessId -eq $PID) { continue }
|
||||
Write-Log "이전 Watchdog의 잔여 프로세스 정리: $($candidate.Name) PID=$($candidate.ProcessId)"
|
||||
Stop-Process -Id $candidate.ProcessId -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
function Get-RotationFilter {
|
||||
@@ -2988,6 +3105,7 @@ function Start-Streams {
|
||||
|
||||
$script:LastRtspHealthCheck = Get-Date
|
||||
$script:RtspHealthFailures = 0
|
||||
$script:StreamStartedAt = Get-Date
|
||||
}
|
||||
|
||||
function Test-StreamsRunning {
|
||||
@@ -3002,6 +3120,18 @@ function Test-StreamsRunning {
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-RtspHealthCheckDue {
|
||||
param([datetime]$Now = (Get-Date))
|
||||
|
||||
if (-not (Test-StreamsRunning)) { return $false }
|
||||
if ($script:StreamStartedAt -eq [DateTime]::MinValue) { return $false }
|
||||
if (($Now - $script:StreamStartedAt).TotalSeconds -lt $RtspHealthStartupGraceSec) {
|
||||
return $false
|
||||
}
|
||||
return (($Now - $script:LastRtspHealthCheck).TotalSeconds -ge
|
||||
$RtspHealthCheckIntervalSec)
|
||||
}
|
||||
|
||||
function New-RtspHealthProbeArguments {
|
||||
param([int]$Channel)
|
||||
|
||||
@@ -3217,14 +3347,16 @@ function Invoke-SimulationTests {
|
||||
Invoke-SimulationCase "RTSP 실제 프레임 검사 설정 파싱" {
|
||||
$path = New-SimulationIni (
|
||||
"[RTSP]`r`nPort=8554`r`nHealthCheckIntervalSec=10`r`n" +
|
||||
"HealthTimeoutSec=5`r`nHealthFailureThreshold=2`r`n"
|
||||
"HealthTimeoutSec=5`r`nHealthFailureThreshold=2`r`n" +
|
||||
"StartupGraceSec=45`r`n"
|
||||
)
|
||||
try {
|
||||
$parsed = Read-IniConfiguration $path
|
||||
Assert-IniSettings $parsed
|
||||
return ($parsed.RTSP.HealthCheckIntervalSec -eq "10" -and
|
||||
$parsed.RTSP.HealthTimeoutSec -eq "5" -and
|
||||
$parsed.RTSP.HealthFailureThreshold -eq "2")
|
||||
$parsed.RTSP.HealthFailureThreshold -eq "2" -and
|
||||
$parsed.RTSP.StartupGraceSec -eq "45")
|
||||
}
|
||||
finally { [System.IO.File]::Delete($path) }
|
||||
}
|
||||
@@ -3538,6 +3670,7 @@ function Invoke-SimulationTests {
|
||||
$arguments -match '-RtspHealthCheckIntervalSec\s+10' -and
|
||||
$arguments -match '-RtspHealthTimeoutSec\s+5' -and
|
||||
$arguments -match '-RtspHealthFailureThreshold\s+2' -and
|
||||
$arguments -match '-RtspHealthStartupGraceSec\s+45' -and
|
||||
$arguments -match '-WatchdogWorker')
|
||||
}
|
||||
|
||||
@@ -3637,6 +3770,42 @@ function Invoke-SimulationTests {
|
||||
$arguments -contains "$RTSPPublishBase/monitor2")
|
||||
}
|
||||
|
||||
Invoke-SimulationCase "RTSP 시작 유예 중에는 프레임 검사를 실행하지 않음" {
|
||||
$savedAudioEnabled = $script:AudioEnabled
|
||||
$savedStreamProcesses = $script:StreamProcesses
|
||||
$savedAudioProcesses = $script:AudioProcesses
|
||||
$savedStartedAt = $script:StreamStartedAt
|
||||
$savedLastCheck = $script:LastRtspHealthCheck
|
||||
try {
|
||||
$now = Get-Date
|
||||
$alive = [PSCustomObject]@{HasExited=$false}
|
||||
$script:AudioEnabled = 0
|
||||
$script:StreamProcesses = @{pipeline=$alive}
|
||||
$script:AudioProcesses = @{}
|
||||
$script:LastRtspHealthCheck = $now.AddHours(-1)
|
||||
$script:StreamStartedAt = $now
|
||||
$blockedDuringGrace = -not (Test-RtspHealthCheckDue -Now $now)
|
||||
$script:StreamStartedAt = $now.AddSeconds(-($RtspHealthStartupGraceSec + 1))
|
||||
$allowedAfterGrace = Test-RtspHealthCheckDue -Now $now
|
||||
return ($blockedDuringGrace -and $allowedAfterGrace)
|
||||
}
|
||||
finally {
|
||||
$script:AudioEnabled = $savedAudioEnabled
|
||||
$script:StreamProcesses = $savedStreamProcesses
|
||||
$script:AudioProcesses = $savedAudioProcesses
|
||||
$script:StreamStartedAt = $savedStartedAt
|
||||
$script:LastRtspHealthCheck = $savedLastCheck
|
||||
}
|
||||
}
|
||||
|
||||
Invoke-SimulationCase "INI FPS 허용 범위 밖 값 거부" {
|
||||
try {
|
||||
Assert-IntegerRange "Video.FrameRate" 0 1 240
|
||||
return $false
|
||||
}
|
||||
catch { return $true }
|
||||
}
|
||||
|
||||
Invoke-SimulationCase "비 HTTPS 의존성 URL 거부" {
|
||||
try { Assert-HttpsDependencyUrl "http://example.com/file.zip" "test"; return $false }
|
||||
catch { return $true }
|
||||
@@ -3751,6 +3920,8 @@ try {
|
||||
$hasMutex = $mutex.WaitOne(0, $false)
|
||||
if (-not $hasMutex) { throw "Screen RTSP Watchdog가 이미 실행 중입니다." }
|
||||
|
||||
Stop-OrphanedPipelineProcesses
|
||||
|
||||
Register-ObjectEvent -InputObject ([Microsoft.Win32.SystemEvents]) `
|
||||
-EventName PowerModeChanged -SourceIdentifier $powerEventSourceIdentifier | Out-Null
|
||||
$powerEventRegistered = $true
|
||||
@@ -3791,10 +3962,11 @@ try {
|
||||
else { Write-Log "NIC 선택: IPv4 기본 경로 자동 감지" }
|
||||
if ($HealthTarget) { Write-Log "통신 확인 대상: $HealthTarget" }
|
||||
Write-Log (
|
||||
"RTSP 프레임 검사: ${RtspHealthCheckIntervalSec}초 주기 / " +
|
||||
"${RtspHealthTimeoutSec}초 제한 / " +
|
||||
"RTSP 프레임 검사: 시작 후 ${RtspHealthStartupGraceSec}초 유예 / " +
|
||||
"${RtspHealthCheckIntervalSec}초 주기 / ${RtspHealthTimeoutSec}초 제한 / " +
|
||||
"${RtspHealthFailureThreshold}회 연속 실패 시 재시작"
|
||||
)
|
||||
Write-Log "영상 설정: ${FrameRate}fps / 채널당 $VideoBitrate"
|
||||
|
||||
$lastSystemSignature = ""
|
||||
$healthySince = $null
|
||||
@@ -3905,6 +4077,7 @@ try {
|
||||
if (($StreamProcesses.Count -gt 0 -or $AudioProcesses.Count -gt 0) -and
|
||||
-not (Test-StreamsRunning)) {
|
||||
Write-Log "FFmpeg 또는 보조 프로세스 비정상 종료 감지"
|
||||
Write-StreamFailureDiagnostics "프로세스 종료 감지"
|
||||
Stop-Streams
|
||||
$healthySince = $null
|
||||
}
|
||||
@@ -3912,9 +4085,7 @@ try {
|
||||
# A live FFmpeg PID does not prove that DDA and both RTSP outputs
|
||||
# are progressing. A new local receiver must decode one frame from
|
||||
# every channel within the configured deadline.
|
||||
if ((Test-StreamsRunning) -and
|
||||
((Get-Date) - $script:LastRtspHealthCheck).TotalSeconds -ge
|
||||
$RtspHealthCheckIntervalSec) {
|
||||
if (Test-RtspHealthCheckDue) {
|
||||
$script:LastRtspHealthCheck = Get-Date
|
||||
$rtspHealth = Test-RtspStreamFrames
|
||||
|
||||
@@ -3934,6 +4105,7 @@ try {
|
||||
|
||||
if ($script:RtspHealthFailures -ge $RtspHealthFailureThreshold) {
|
||||
Write-Log "RTSP 송출 정지 감지 - 전체 AV 파이프라인 재시작"
|
||||
Write-StreamFailureDiagnostics $rtspHealth.Reason
|
||||
Stop-Streams
|
||||
$healthySince = $null
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user