Initial commit: split dlp-io library out of DataPacker

This commit is contained in:
2026-07-31 14:58:37 +08:00
commit 3b0c81fd9d
22 changed files with 2509 additions and 0 deletions
+453
View File
@@ -0,0 +1,453 @@
name: publish-dlp-io
# Publish the standalone dlp-io distribution from an immutable dlp-io-vX.Y.Z tag.
# The Windows runner cannot reach github.com, so this workflow uses no third-party actions.
on:
push:
tags:
- "dlp-io-v*.*.*"
workflow_dispatch:
inputs:
version:
description: "Immutable package tag, for example dlp-io-v0.1.0"
required: true
type: string
env:
PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
jobs:
publish:
runs-on: windows-latest
defaults:
run:
working-directory: repo
steps:
- name: Clone immutable package tag
shell: pwsh
working-directory: .
env:
ACTION_EVENT_NAME: ${{ github.event_name }}
ACTION_REF_NAME: ${{ github.ref_name }}
ACTION_SERVER_URL: ${{ github.server_url }}
ACTION_REPOSITORY: ${{ github.repository }}
ACTION_CLONE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DLP_IO_INPUT_VERSION: ${{ inputs.version }}
run: |
$ErrorActionPreference = "Stop"
$tag = if ($env:ACTION_EVENT_NAME -eq "workflow_dispatch") {
$env:DLP_IO_INPUT_VERSION
} else {
$env:ACTION_REF_NAME
}
if ($tag -notmatch '^dlp-io-v\d+\.\d+\.\d+$') {
throw "Package tag must use dlp-io-vMAJOR.MINOR.PATCH format: $tag"
}
$packageVersion = $tag.Substring("dlp-io-v".Length)
$token = $env:ACTION_CLONE_TOKEN
if ([string]::IsNullOrWhiteSpace($token)) {
throw "GITHUB_TOKEN is required to clone the repository"
}
$server = [Uri]$env:ACTION_SERVER_URL
$clone = "$($server.Scheme)://x-access-token:$token@$($server.Authority)/$($env:ACTION_REPOSITORY).git"
$masked = [regex]::Escape($token)
$tagRef = "refs/tags/$tag"
$peeledRef = "$tagRef^{}"
$remoteOutput = @(git ls-remote --tags $clone $tagRef $peeledRef 2>&1)
$remoteExit = $LASTEXITCODE
$remoteOutput | ForEach-Object { $_ -replace $masked, "***" }
if ($remoteExit -ne 0) { throw "remote tag lookup failed: $remoteExit" }
$remoteRefs = @{}
foreach ($line in $remoteOutput) {
if ($line -match '^([0-9a-f]{40})\s+(.+)$') {
$remoteRefs[$Matches[2]] = $Matches[1]
}
}
if (-not $remoteRefs.ContainsKey($tagRef)) {
throw "Remote package tag does not exist: $tagRef"
}
if (-not $remoteRefs.ContainsKey($peeledRef)) {
throw "Package tag must be annotated: $tagRef"
}
$remoteCommit = $remoteRefs[$peeledRef]
git clone -c core.autocrlf=true --depth 1 --branch $tag $clone repo 2>&1 |
ForEach-Object { $_ -replace $masked, "***" }
if ($LASTEXITCODE -ne 0) { throw "tagged source clone failed: $LASTEXITCODE" }
Push-Location repo
try {
$sourceVersion = (& py -3.13 -c "import dlp_io; print(dlp_io.__version__)").Trim()
} finally {
Pop-Location
}
if ($sourceVersion -ne $packageVersion) {
throw "dlp-io tag/version mismatch: tag=$tag package=$sourceVersion"
}
$tagCommit = (git -C repo rev-list -n 1 $tag).Trim()
$headCommit = (git -C repo rev-parse HEAD).Trim()
if ($tagCommit -ne $remoteCommit -or $remoteCommit -ne $headCommit) {
throw "Immutable tag checkout mismatch: remote=$remoteCommit local=$tagCommit HEAD=$headCommit"
}
"DLP_IO_PACKAGE_TAG=$tag" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append
"DLP_IO_PACKAGE_VERSION=$packageVersion" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append
Write-Host "package source: $tag @ $headCommit"
- name: Test Python 3.9-3.13
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
foreach ($pythonVersion in @("3.9", "3.10", "3.11", "3.12", "3.13")) {
& py "-$pythonVersion" --version
if ($LASTEXITCODE -ne 0) { throw "Python $pythonVersion is required" }
& py "-$pythonVersion" -m pip install -r requirements-dev.txt --progress-bar off
if ($LASTEXITCODE -ne 0) { throw "test dependency install failed on $pythonVersion" }
$abi = $pythonVersion.Replace(".", "")
$baseTemp = Join-Path $env:TEMP "dlp-io-py$abi-$PID"
& py "-$pythonVersion" -m pytest -q --basetemp $baseTemp
if ($LASTEXITCODE -ne 0) { throw "full pytest failed on $pythonVersion" }
}
- name: Build and check distribution
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
py -3.13 -m pip install build twine --progress-bar off
if ($LASTEXITCODE -ne 0) { throw "build dependency install failed" }
py -3.13 -m build
if ($LASTEXITCODE -ne 0) { throw "dlp-io build failed" }
py -3.13 -m twine check "dist\dlp_io-$($env:DLP_IO_PACKAGE_VERSION)*"
if ($LASTEXITCODE -ne 0) { throw "twine check failed" }
$version = $env:DLP_IO_PACKAGE_VERSION
$expected = @(
"dlp_io-$version-py3-none-any.whl",
"dlp_io-$version.tar.gz"
)
$actual = @(Get-ChildItem -LiteralPath dist -File | ForEach-Object Name)
if (Compare-Object $expected $actual) {
throw "distribution file set mismatch: $($actual -join ', ')"
}
- name: Smoke test isolated wheel
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
$root = Join-Path $env:TEMP ("dlp-io-wheel-smoke-" + $PID)
$rootFull = [IO.Path]::GetFullPath($root)
$tempFull = [IO.Path]::GetFullPath($env:TEMP).TrimEnd('\') + '\'
if (-not $rootFull.StartsWith($tempFull, [StringComparison]::OrdinalIgnoreCase)) {
throw "unsafe wheel smoke path: $rootFull"
}
try {
py -3.13 -m venv $rootFull
if ($LASTEXITCODE -ne 0) { throw "wheel smoke venv creation failed" }
$python = Join-Path $rootFull "Scripts\python.exe"
$dist = (Resolve-Path dist).Path
& $python -m pip install --no-index --find-links $dist "dlp-io==$env:DLP_IO_PACKAGE_VERSION" --progress-bar off
if ($LASTEXITCODE -ne 0) { throw "wheel installation failed" }
Push-Location $rootFull
try {
& $python -c "import dlp_io,sys,tempfile; from pathlib import Path; assert dlp_io.__version__ == '$env:DLP_IO_PACKAGE_VERSION'; td=tempfile.TemporaryDirectory(); p=Path(td.name)/'payload.bin'; w=dlp_io.open(p,'wb'); assert w.write(b'0123456789') == 10; w.close(); dlp_io.configure(python_executable=sys.executable); r=dlp_io.open(p,'rb'); assert r.read(4) == b'0123'; assert r.seek(-3,2) == 7; assert r.read() == b'789'; r.close(); dlp_io.shutdown(); td.cleanup(); print('wheel smoke passed')"
if ($LASTEXITCODE -ne 0) { throw "wheel smoke failed" }
} finally {
Pop-Location
}
} finally {
if (Test-Path -LiteralPath $rootFull) {
Remove-Item -LiteralPath $rootFull -Recurse -Force
}
}
- name: Upload package to Gitea PyPI
shell: pwsh
env:
TWINE_USERNAME: antior
TWINE_PASSWORD: ${{ secrets.CI_PACKAGE_TOKEN }}
TWINE_REPOSITORY_URL: https://gitea.docker.antior.cn/api/packages/antior/pypi
run: |
$ErrorActionPreference = "Stop"
if ([string]::IsNullOrWhiteSpace($env:TWINE_PASSWORD)) {
throw "CI_PACKAGE_TOKEN repository secret is required"
}
function Get-RegistryEntries([string]$SimpleUrl, [hashtable]$Headers) {
try {
$response = Invoke-WebRequest -Headers $Headers -Uri $SimpleUrl
} catch {
if ([int]$_.Exception.Response.StatusCode -eq 404) { return @() }
throw
}
$pattern = '<a\s+href="(?<href>[^"]+)"[^>]*>(?<name>[^<]+)</a>'
foreach ($match in [regex]::Matches($response.Content, $pattern, "IgnoreCase")) {
$href = [Net.WebUtility]::HtmlDecode($match.Groups["href"].Value)
$uri = [Uri]$href
$hashMatch = [regex]::Match($uri.Fragment, '^#sha256=(?<hash>[0-9a-fA-F]{64})$')
if (-not $uri.IsAbsoluteUri -or -not $hashMatch.Success) {
throw "registry simple index contains an invalid artifact link"
}
$builder = [UriBuilder]$uri
$builder.Fragment = ""
[pscustomobject]@{
filename = [Net.WebUtility]::HtmlDecode($match.Groups["name"].Value)
url = $builder.Uri.AbsoluteUri
sha256 = $hashMatch.Groups["hash"].Value.ToLowerInvariant()
}
}
}
function Get-ArchiveContentDigest([string]$Path) {
$script = @'
import hashlib
import sys
import tarfile
import zipfile
from pathlib import Path, PurePosixPath
path = Path(sys.argv[1])
entries = []
if path.suffix == ".whl":
with zipfile.ZipFile(path) as archive:
entries = [
(item.filename, archive.read(item))
for item in archive.infolist()
if not item.is_dir()
]
else:
with tarfile.open(path, "r:*") as archive:
for item in archive.getmembers():
if not item.isfile():
continue
stream = archive.extractfile(item)
parts = PurePosixPath(item.name).parts[1:]
if stream is not None and parts:
entries.append(("/".join(parts), stream.read()))
digest = hashlib.sha256()
for name, content in sorted(entries):
digest.update(name.encode("utf-8"))
digest.update(b"\0")
digest.update(len(content).to_bytes(8, "big"))
digest.update(hashlib.sha256(content).digest())
print(digest.hexdigest())
'@
$digest = (& py -3.13 -c $script $Path).Trim()
if ($LASTEXITCODE -ne 0 -or $digest -notmatch '^[0-9a-f]{64}$') {
throw "failed to calculate archive content digest: $Path"
}
return $digest
}
$packageVersion = $env:DLP_IO_PACKAGE_VERSION
$expected = @(
"dlp_io-$packageVersion-py3-none-any.whl",
"dlp_io-$packageVersion.tar.gz"
)
$headers = @{ Authorization = "token $env:TWINE_PASSWORD" }
$simpleUrl = "$env:TWINE_REPOSITORY_URL/simple/dlp-io/"
$registryOrigin = [Uri]$env:TWINE_REPOSITORY_URL
$entries = @(Get-RegistryEntries $simpleUrl $headers)
$missingArtifacts = [System.Collections.Generic.List[string]]::new()
$root = Join-Path $env:TEMP ("dlp-io-upload-state-" + [Guid]::NewGuid().ToString("N"))
$rootFull = [IO.Path]::GetFullPath($root)
$tempFull = [IO.Path]::GetFullPath($env:TEMP).TrimEnd('\') + '\'
if (-not $rootFull.StartsWith($tempFull, [StringComparison]::OrdinalIgnoreCase)) {
throw "unsafe upload verification path: $rootFull"
}
try {
New-Item -ItemType Directory -Force $rootFull | Out-Null
foreach ($filename in $expected) {
$local = (Resolve-Path (Join-Path dist $filename)).Path
$entry = @($entries | Where-Object filename -eq $filename)
if ($entry.Count -eq 0) {
$missingArtifacts.Add($local)
continue
}
if ($entry.Count -ne 1) {
throw "registry simple index contains duplicate artifact links: $filename"
}
$downloadUri = [Uri]$entry[0].url
if (-not $downloadUri.Scheme.Equals($registryOrigin.Scheme, [StringComparison]::OrdinalIgnoreCase) -or
-not $downloadUri.Authority.Equals($registryOrigin.Authority, [StringComparison]::OrdinalIgnoreCase)) {
throw "download URL is not on the package registry origin: $filename"
}
$remote = Join-Path $rootFull $filename
Invoke-WebRequest -Headers $headers -Uri $downloadUri.AbsoluteUri -OutFile $remote
$remoteHash = (Get-FileHash -LiteralPath $remote -Algorithm SHA256).Hash.ToLowerInvariant()
if ($remoteHash -ne $entry[0].sha256) {
throw "registry download hash mismatch: $filename"
}
if ((Get-ArchiveContentDigest $local) -ne (Get-ArchiveContentDigest $remote)) {
throw "existing registry artifact differs from tagged build: $filename"
}
Write-Host "registry artifact already matches tagged build: $filename"
}
if ($missingArtifacts.Count -gt 0) {
$twineArgs = @(
"-3.13", "-m", "twine", "upload", "--non-interactive",
"--disable-progress-bar", "--repository-url", $env:TWINE_REPOSITORY_URL
) + $missingArtifacts
& py @twineArgs
if ($LASTEXITCODE -ne 0) { throw "twine upload failed" }
} else {
Write-Host "all registry artifacts already match the tagged build"
}
} finally {
if (Test-Path -LiteralPath $rootFull) {
Remove-Item -LiteralPath $rootFull -Recurse -Force
}
}
- name: Test registry package files
shell: pwsh
env:
REGISTRY_TOKEN: ${{ secrets.CI_PACKAGE_TOKEN }}
TWINE_REPOSITORY_URL: https://gitea.docker.antior.cn/api/packages/antior/pypi
run: |
$ErrorActionPreference = "Stop"
$packageVersion = $env:DLP_IO_PACKAGE_VERSION
$headers = @{ Authorization = "token $env:REGISTRY_TOKEN" }
function Get-RegistryEntries([string]$SimpleUrl, [hashtable]$Headers) {
try {
$response = Invoke-WebRequest -Headers $Headers -Uri $SimpleUrl
} catch {
if ([int]$_.Exception.Response.StatusCode -eq 404) { return @() }
throw
}
$pattern = '<a\s+href="(?<href>[^"]+)"[^>]*>(?<name>[^<]+)</a>'
foreach ($match in [regex]::Matches($response.Content, $pattern, "IgnoreCase")) {
$href = [Net.WebUtility]::HtmlDecode($match.Groups["href"].Value)
$uri = [Uri]$href
$hashMatch = [regex]::Match($uri.Fragment, '^#sha256=(?<hash>[0-9a-fA-F]{64})$')
if (-not $uri.IsAbsoluteUri -or -not $hashMatch.Success) {
throw "registry simple index contains an invalid artifact link"
}
$builder = [UriBuilder]$uri
$builder.Fragment = ""
[pscustomobject]@{
filename = [Net.WebUtility]::HtmlDecode($match.Groups["name"].Value)
url = $builder.Uri.AbsoluteUri
sha256 = $hashMatch.Groups["hash"].Value.ToLowerInvariant()
}
}
}
function Get-ArchiveContentDigest([string]$Path) {
$script = @'
import hashlib
import sys
import tarfile
import zipfile
from pathlib import Path, PurePosixPath
path = Path(sys.argv[1])
entries = []
if path.suffix == ".whl":
with zipfile.ZipFile(path) as archive:
entries = [
(item.filename, archive.read(item))
for item in archive.infolist()
if not item.is_dir()
]
else:
with tarfile.open(path, "r:*") as archive:
for item in archive.getmembers():
if not item.isfile():
continue
stream = archive.extractfile(item)
parts = PurePosixPath(item.name).parts[1:]
if stream is not None and parts:
entries.append(("/".join(parts), stream.read()))
digest = hashlib.sha256()
for name, content in sorted(entries):
digest.update(name.encode("utf-8"))
digest.update(b"\0")
digest.update(len(content).to_bytes(8, "big"))
digest.update(hashlib.sha256(content).digest())
print(digest.hexdigest())
'@
$digest = (& py -3.13 -c $script $Path).Trim()
if ($LASTEXITCODE -ne 0 -or $digest -notmatch '^[0-9a-f]{64}$') {
throw "failed to calculate archive content digest: $Path"
}
return $digest
}
$expected = @(
"dlp_io-$packageVersion-py3-none-any.whl",
"dlp_io-$packageVersion.tar.gz"
)
$simpleUrl = "$env:TWINE_REPOSITORY_URL/simple/dlp-io/"
$entries = @()
$actual = @()
for ($attempt = 1; $attempt -le 20; $attempt++) {
$entries = @(Get-RegistryEntries $simpleUrl $headers)
$actual = @($entries | ForEach-Object filename)
if (-not (Compare-Object $expected $actual)) { break }
if ($attempt -lt 20) { Start-Sleep -Seconds 3 }
}
if (Compare-Object $expected $actual) {
throw "registry package file mismatch: $($actual -join ', ')"
}
$registryRoot = Join-Path $env:TEMP ("dlp-io-registry-smoke-" + $PID)
$registryRootFull = [IO.Path]::GetFullPath($registryRoot)
$tempFull = [IO.Path]::GetFullPath($env:TEMP).TrimEnd('\') + '\'
if (-not $registryRootFull.StartsWith($tempFull, [StringComparison]::OrdinalIgnoreCase)) {
throw "unsafe registry smoke path: $registryRootFull"
}
try {
$downloads = Join-Path $registryRootFull "downloads"
New-Item -ItemType Directory -Force $downloads | Out-Null
$registryOrigin = [Uri]$env:TWINE_REPOSITORY_URL
foreach ($filename in $expected) {
$entry = @($entries | Where-Object filename -eq $filename)
if ($entry.Count -ne 1 -or [string]::IsNullOrWhiteSpace($entry[0].url)) {
throw "registry simple index is missing a download URL: $filename"
}
if ([string]::IsNullOrWhiteSpace($entry[0].sha256)) {
throw "registry simple index is missing SHA-256: $filename"
}
$downloadUri = [Uri]$entry[0].url
if (-not $downloadUri.IsAbsoluteUri -or
-not $downloadUri.Scheme.Equals($registryOrigin.Scheme, [StringComparison]::OrdinalIgnoreCase) -or
-not $downloadUri.Authority.Equals($registryOrigin.Authority, [StringComparison]::OrdinalIgnoreCase)) {
throw "download URL is not on the package registry origin: $filename"
}
$target = Join-Path $downloads $filename
Invoke-WebRequest -Headers $headers -Uri $downloadUri.AbsoluteUri -OutFile $target
$downloadHash = (Get-FileHash -LiteralPath $target -Algorithm SHA256).Hash.ToLowerInvariant()
if ($downloadHash -ne $entry[0].sha256) {
throw "registry download hash mismatch: $filename"
}
$local = (Resolve-Path (Join-Path dist $filename)).Path
if ((Get-ArchiveContentDigest $local) -ne (Get-ArchiveContentDigest $target)) {
throw "registry artifact differs from tagged build: $filename"
}
}
$venv = Join-Path $registryRootFull "venv"
py -3.13 -m venv $venv
if ($LASTEXITCODE -ne 0) { throw "registry smoke venv creation failed" }
$python = Join-Path $venv "Scripts\python.exe"
$registryWheel = Join-Path $downloads "dlp_io-$packageVersion-py3-none-any.whl"
& $python -m pip install --no-index --no-deps $registryWheel --progress-bar off
if ($LASTEXITCODE -ne 0) { throw "registry package installation failed" }
Push-Location $registryRootFull
try {
& $python -c "import dlp_io,sys,tempfile; from pathlib import Path; assert dlp_io.__version__ == '$packageVersion'; td=tempfile.TemporaryDirectory(); p=Path(td.name)/'payload.bin'; w=dlp_io.open(p,'wb'); w.write(b'registry'); w.close(); dlp_io.configure(python_executable=sys.executable); r=dlp_io.open(p,'rb'); assert r.read() == b'registry'; r.close(); dlp_io.shutdown(); td.cleanup(); print('registry install passed')"
if ($LASTEXITCODE -ne 0) { throw "registry installation smoke failed" }
} finally {
Pop-Location
}
} finally {
if (Test-Path -LiteralPath $registryRootFull) {
Remove-Item -LiteralPath $registryRootFull -Recurse -Force
}
}
Write-Host "registry verified: dlp-io==$packageVersion ($($actual.Count) files)"