ai-tester
44473a1055
Repository Validate / repository-checks (push) Successful in 21s
Co-authored-by: AI Tester <ai-tester@git.p2prod.ru>
330 lines
17 KiB
YAML
330 lines
17 KiB
YAML
name: Repository Validate
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- master
|
|
- main
|
|
pull_request:
|
|
branches:
|
|
- master
|
|
- main
|
|
- task/**
|
|
|
|
jobs:
|
|
repository-checks:
|
|
runs-on: agent-environment
|
|
steps:
|
|
- name: Checkout
|
|
shell: bash
|
|
env:
|
|
CHECKOUT_SERVER_URL: ${{ github.server_url }}
|
|
CHECKOUT_REPOSITORY: ${{ github.repository }}
|
|
CHECKOUT_SHA: ${{ github.sha }}
|
|
CHECKOUT_TOKEN: ${{ github.token }}
|
|
run: |
|
|
python3 - <<'CHECKOUT_PY'
|
|
"""Self-contained checkout embedded in generated CI; no downloaded actions."""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import subprocess
|
|
from urllib.parse import urlsplit
|
|
|
|
|
|
def checkout(environment=None):
|
|
env = dict(os.environ if environment is None else environment)
|
|
server = env.pop('CHECKOUT_SERVER_URL', '').rstrip('/')
|
|
repo = env.pop('CHECKOUT_REPOSITORY', '')
|
|
sha = env.pop('CHECKOUT_SHA', '')
|
|
token = env.pop('CHECKOUT_TOKEN', '')
|
|
parsed = urlsplit(server)
|
|
if (parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username
|
|
or parsed.password or parsed.query or parsed.fragment):
|
|
raise ValueError('invalid checkout server URL')
|
|
if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', repo) or any(p in ('.', '..') for p in repo.split('/')):
|
|
raise ValueError('invalid checkout repository')
|
|
if not re.fullmatch(r'[0-9a-fA-F]{40}|[0-9a-fA-F]{64}', sha):
|
|
raise ValueError('checkout requires an exact commit SHA')
|
|
workspace = Path(env.get('GITHUB_WORKSPACE') or env.get('GITEA_WORKSPACE') or os.getcwd()).resolve()
|
|
workspace.mkdir(parents=True, exist_ok=True)
|
|
# Never inherit authentication overrides, tracing, hooks or checkout filters.
|
|
env = {k: v for k, v in env.items() if not k.startswith('GIT_')}
|
|
env.update(GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL='/dev/null',
|
|
GIT_TERMINAL_PROMPT='0', GIT_LFS_SKIP_SMUDGE='1')
|
|
if (workspace / '.git').exists():
|
|
raise ValueError('checkout requires a fresh workspace')
|
|
|
|
def git(*args, auth=False):
|
|
child = dict(env)
|
|
if auth and token:
|
|
credential = base64.b64encode(('x-access-token:' + token).encode()).decode()
|
|
child.update(GIT_CONFIG_COUNT='1',
|
|
GIT_CONFIG_KEY_0='http.extraHeader',
|
|
GIT_CONFIG_VALUE_0='Authorization: Basic ' + credential)
|
|
result = subprocess.run(['git', '-c', 'core.hooksPath=/dev/null', '-C', str(workspace), *args],
|
|
env=child, capture_output=True, text=True, timeout=180)
|
|
if result.returncode:
|
|
# Transport output may contain credentials or upstream-controlled data.
|
|
raise RuntimeError('checkout git operation failed: ' + args[0])
|
|
return result.stdout.strip()
|
|
|
|
git('init')
|
|
git('remote', 'add', 'origin', server + '/' + repo + '.git')
|
|
git('-c', 'http.followRedirects=false', 'fetch', '--no-tags', '--depth=1', 'origin', sha, auth=True)
|
|
actual = git('rev-parse', 'FETCH_HEAD^{commit}')
|
|
if actual.lower() != sha.lower():
|
|
raise ValueError('fetched commit does not match requested SHA')
|
|
git('checkout', '--detach', '--force', actual)
|
|
if git('rev-parse', 'HEAD').lower() != sha.lower():
|
|
raise ValueError('checked out commit does not match requested SHA')
|
|
print('Checked out requested commit ' + actual)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
checkout()
|
|
except (ValueError, RuntimeError, subprocess.TimeoutExpired) as exc:
|
|
print('AGENT_CI_RESULT command_id=checkout stage=checkout owner=infrastructure '
|
|
'repair_role=operator exit_code=1 expected_exit_code=0 status=failed')
|
|
# Do not stringify subprocess exceptions: they may include command output.
|
|
raise SystemExit('Checkout failed (' + type(exc).__name__ + ')') from None
|
|
CHECKOUT_PY
|
|
|
|
- name: Validate facade structure
|
|
shell: bash
|
|
run: |
|
|
python3 - <<'PY'
|
|
from pathlib import Path
|
|
|
|
path = Path(".agent/project.yaml")
|
|
if not path.is_file():
|
|
raise SystemExit(".agent/project.yaml is missing")
|
|
text = path.read_text(encoding="utf-8")
|
|
for key in ["schema_version", "registry", "project", "components"]:
|
|
if not any(line.startswith(key + ":") for line in text.splitlines()):
|
|
raise SystemExit(f".agent/project.yaml missing required key: {key}")
|
|
if not any(line.lstrip().startswith("- ") for line in text.splitlines()):
|
|
raise SystemExit(".agent/project.yaml must declare at least one component")
|
|
print("facade structure ok")
|
|
PY
|
|
|
|
- name: Run facade commands
|
|
shell: bash
|
|
run: |
|
|
set -euo pipefail
|
|
full_run=true
|
|
if [ "${GITHUB_EVENT_NAME:-}" = "pull_request" ] && [[ "${GITHUB_BASE_REF:-}" == task/* ]]; then
|
|
full_run=false
|
|
fi
|
|
export PATH="$PWD/.venv/bin:$PATH"
|
|
printf 'AGENT_CI_MODE full_run=%s event=%s base_ref=%s\n' "$full_run" "${GITHUB_EVENT_NAME:-unknown}" "${GITHUB_BASE_REF:-}"
|
|
run_ci_stage() {
|
|
command_id="$1"
|
|
stage="$2"
|
|
owner="$3"
|
|
repair_role="$4"
|
|
command="$5"
|
|
expected_exit_code="$6"
|
|
missing_policy="$7"
|
|
shift 7
|
|
missing_paths=()
|
|
for required_path in "$@"; do
|
|
if [ ! -e "$required_path" ]; then
|
|
missing_paths+=("$required_path")
|
|
fi
|
|
done
|
|
if [ "${#missing_paths[@]}" -gt 0 ]; then
|
|
missing_csv="$(IFS=,; echo "${missing_paths[*]}")"
|
|
if [ "$full_run" != true ] && [ "$missing_policy" = skip_partial ]; then
|
|
printf 'AGENT_CI_RESULT command_id=%s stage=%s owner=%s repair_role=%s exit_code=0 expected_exit_code=%s status=skipped_missing_prerequisites missing_paths=%s\n' "$command_id" "$stage" "$owner" "$repair_role" "$expected_exit_code" "$missing_csv"
|
|
return 0
|
|
fi
|
|
printf 'AGENT_CI_RESULT command_id=%s stage=%s owner=%s repair_role=%s exit_code=97 expected_exit_code=%s status=missing_prerequisites missing_paths=%s\n' "$command_id" "$stage" "$owner" "$repair_role" "$expected_exit_code" "$missing_csv"
|
|
return 97
|
|
fi
|
|
if [ -z "$command" ]; then
|
|
echo "skip $command_id (not declared)"
|
|
return 0
|
|
fi
|
|
echo "::group::$command_id"
|
|
set +e
|
|
bash -o pipefail -eu -c "$command"
|
|
exit_code="$?"
|
|
set -e
|
|
# Executor reserves 78 for tracked preparation-output conflicts.
|
|
# Other dependency failures retain infrastructure ownership.
|
|
if [ "$stage" = dependency_preparation ] && [ "$exit_code" -eq 78 ]; then
|
|
owner=project
|
|
repair_role=developer
|
|
fi
|
|
status=failed
|
|
if [ "$exit_code" -eq "$expected_exit_code" ]; then
|
|
status=passed
|
|
fi
|
|
if [ "$status" = failed ]; then
|
|
python3 - "$command_id" "$exit_code" <<'CI_DIAGNOSTICS_PY' || echo 'AGENT_CI_DIAGNOSTICS_ARTIFACT collector_failed'
|
|
"""Bounded, opt-in CI failure reports; embedded in the canonical workflow.
|
|
|
|
Gitea Actions artifact v3 protocol (no external Action or package dependency).
|
|
"""
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import PurePosixPath
|
|
import re
|
|
import stat
|
|
import sys
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
FILE_LIMIT = 65536
|
|
FILE_COUNT = 8
|
|
MANIFEST = '.ci/diagnostics.json'
|
|
|
|
|
|
def read_relative(root, name, limit=FILE_LIMIT):
|
|
path = PurePosixPath(name)
|
|
if not name or path.is_absolute() or any(p in ('', '.', '..') for p in name.split('/')):
|
|
raise ValueError('invalid path')
|
|
fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY)
|
|
try:
|
|
for part in path.parts[:-1]:
|
|
child = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=fd)
|
|
os.close(fd)
|
|
fd = child
|
|
child = os.open(path.name, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd)
|
|
with os.fdopen(child, 'rb') as stream:
|
|
if not stat.S_ISREG(os.fstat(stream.fileno()).st_mode):
|
|
raise ValueError('not a regular file')
|
|
return stream.read(limit + 1)
|
|
finally:
|
|
os.close(fd)
|
|
|
|
|
|
def scrub(text, env):
|
|
for key, value in env.items():
|
|
if len(value) >= 8 and any(word in key.upper() for word in ('TOKEN', 'SECRET', 'PASSWORD', 'API_KEY')):
|
|
text = text.replace(value, '<redacted>')
|
|
return text
|
|
|
|
|
|
def collect(root, command, exit_code, env):
|
|
report = {'version': 1, 'command_id': command, 'exit_code': exit_code,
|
|
'sha': env.get('GITHUB_SHA', ''), 'files': []}
|
|
try:
|
|
raw = read_relative(root, MANIFEST)
|
|
if len(raw) > FILE_LIMIT:
|
|
raise ValueError('manifest too large')
|
|
spec = json.loads(raw)
|
|
if spec.get('version') != 1 or not isinstance(spec.get('commands'), dict):
|
|
raise ValueError('invalid manifest')
|
|
names = spec['commands'].get(command, [])
|
|
if not isinstance(names, list) or len(names) > FILE_COUNT or any(not isinstance(n, str) for n in names):
|
|
raise ValueError('invalid file list')
|
|
report['manifest_status'] = 'loaded'
|
|
except FileNotFoundError:
|
|
report['manifest_status'] = 'missing'
|
|
names = []
|
|
except (OSError, ValueError, TypeError, AttributeError):
|
|
report['manifest_status'] = 'invalid'
|
|
names = []
|
|
for name in names:
|
|
item = {'path': name}
|
|
try:
|
|
# No directory traversal, symlinks, devices, hidden credential files,
|
|
# wildcards or recursive workspace scans.
|
|
if any(p.startswith('.') for p in PurePosixPath(name).parts) or len(name) > 512:
|
|
raise ValueError('hidden or invalid path')
|
|
raw = read_relative(root, name)
|
|
item.update(status='captured', truncated=len(raw) > FILE_LIMIT,
|
|
content=scrub(raw[:FILE_LIMIT].decode('utf-8', errors='replace'), env))
|
|
except (OSError, ValueError):
|
|
item['status'] = 'unavailable'
|
|
report['files'].append(item)
|
|
return report
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, *args, **kwargs):
|
|
return None
|
|
|
|
|
|
def upload(report, env, request=None):
|
|
base = env.get('ACTIONS_RUNTIME_URL', '')
|
|
token = env.get('ACTIONS_RUNTIME_TOKEN', '')
|
|
run = env.get('GITHUB_RUN_ID', '')
|
|
parsed = urllib.parse.urlsplit(base)
|
|
if not base or not token or not run:
|
|
return 'runtime_unavailable'
|
|
if parsed.scheme not in ('http', 'https') or not parsed.netloc or parsed.username or parsed.password or parsed.query or parsed.fragment:
|
|
raise ValueError('invalid runtime URL')
|
|
if request is None:
|
|
opener = urllib.request.build_opener(NoRedirect)
|
|
def request(method, url, body, headers):
|
|
req = urllib.request.Request(url, data=body, method=method, headers=headers)
|
|
try:
|
|
with opener.open(req, timeout=10) as response:
|
|
return response.read(65536)
|
|
except urllib.error.HTTPError as exc:
|
|
raise RuntimeError(f'artifact {method} HTTP {exc.code}') from None
|
|
endpoint = base.rstrip('/')+'/_apis/pipelines/workflows/'+urllib.parse.quote(run, safe='')+'/artifacts'
|
|
name = 'ci-failure-'+re.sub('[^A-Za-z0-9_-]', '-', report['command_id'])[:60]+'-'+re.sub('[^0-9]', '', env.get('GITHUB_RUN_ATTEMPT', '1'))[:8]
|
|
headers = {'Authorization': 'Bearer '+token, 'Content-Type': 'application/json'}
|
|
created = request('POST', endpoint+'?api-version=6.0-preview', json.dumps({'Type': 'actions_storage', 'Name': name}).encode(), headers)
|
|
location = json.loads(created)['fileContainerResourceUrl']
|
|
location = urllib.parse.urljoin(endpoint, location)
|
|
target = urllib.parse.urlsplit(location)
|
|
if (target.scheme, target.netloc) != (parsed.scheme, parsed.netloc) or target.username or target.password:
|
|
raise ValueError('cross-origin artifact URL')
|
|
data = json.dumps(report, ensure_ascii=True).encode()
|
|
url = location+('&' if target.query else '?')+urllib.parse.urlencode({'itemPath': name+'/diagnostics.json'})
|
|
request('PUT', url, data, {**headers, 'Content-Type': 'application/octet-stream',
|
|
'Content-Range': f'bytes 0-{len(data)-1}/{len(data)}', 'x-tfs-filelength': str(len(data)),
|
|
'x-actions-results-md5': base64.b64encode(hashlib.md5(data).digest()).decode('ascii')})
|
|
request('PATCH', endpoint+'?'+urllib.parse.urlencode({'artifactName': name, 'api-version': '6.0-preview'}),
|
|
json.dumps({'Size': len(data)}).encode(), headers)
|
|
return 'uploaded:'+name
|
|
|
|
|
|
def main():
|
|
report = collect(os.getcwd(), sys.argv[1], int(sys.argv[2]), os.environ)
|
|
# A bounded preview remains in the job log even if artifact upload fails.
|
|
preview = json.loads(json.dumps(report))
|
|
for item in preview['files']:
|
|
if 'content' in item:
|
|
item['content'] = item['content'][:1000]
|
|
print('AGENT_CI_DIAGNOSTICS '+json.dumps(preview, ensure_ascii=True))
|
|
try:
|
|
result = upload(report, os.environ)
|
|
except Exception as exc:
|
|
result = 'upload_failed:'+str(exc) if isinstance(exc, RuntimeError) else 'upload_failed:'+type(exc).__name__
|
|
print('AGENT_CI_DIAGNOSTICS_ARTIFACT '+result)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|
|
CI_DIAGNOSTICS_PY
|
|
fi
|
|
printf 'AGENT_CI_RESULT command_id=%s stage=%s owner=%s repair_role=%s exit_code=%s expected_exit_code=%s status=%s\n' "$command_id" "$stage" "$owner" "$repair_role" "$exit_code" "$expected_exit_code" "$status"
|
|
echo "::endgroup::"
|
|
if [ "$status" = passed ]; then
|
|
return 0
|
|
fi
|
|
if [ "$exit_code" -eq 0 ]; then
|
|
return 1
|
|
fi
|
|
return "$exit_code"
|
|
}
|
|
run_ci_stage 'tools.dependencies.resolution' 'dependency_prerequisite' 'project' 'developer' '' '0' 'fail' 'uv.lock'
|
|
run_ci_stage 'tools.dependencies.restore' 'dependency_preparation' 'infrastructure' 'infrastructure' '/opt/agent-environment/bin/python -I -m agent_environment_executor.cli --workspace "$PWD" --component '"'"'tools'"'"' --mode restore' '0' 'fail'
|
|
run_ci_stage 'tools.validate' 'command' 'project' 'developer' 'python -m compileall tools tests' '0' 'skip_partial' 'pyproject.toml' 'tools' 'tests'
|
|
run_ci_stage 'tools.test' 'command' 'project' 'developer' 'pytest' '0' 'skip_partial' 'pyproject.toml' 'tests/test_example.py'
|
|
run_ci_stage 'tools.smoke' 'command' 'project' 'developer' 'python -m tools.example examples/input.json' '0' 'skip_partial' 'pyproject.toml' 'tools/example.py' 'examples/input.json'
|
|
run_ci_stage 'acceptance.help-command' 'acceptance' 'project' 'developer' 'python -m tools.example --help' '0' 'skip_partial' 'examples/input.json' 'tools/example.py'
|