Pulse/scripts/tests/test_api_failure_index.py
pulse-triage[bot] 5be9e06ca1 Keep API failure collection fatal with optional build caching
The old whole-job assertion rejected the narrowly optional cache restore even though API execution, admission and sidecar upload remain mandatory. Check every step structurally and allow only the complete pinned cache-restore shape. Adverse controls reject masking the job, runtime, admission or upload, and reject cache lookalike actions, arbitrary execution, paths and conditions.

Change-source: pulse-maintainer
2026-10-31 16:08:07 +00:00

594 lines
34 KiB
Python

#!/usr/bin/env python3
"""Shard artifacts expose bounded source identities, never assertion payloads."""
import copy
import hashlib
import json
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
import unittest
import yaml
ROOT = Path(__file__).resolve().parents[2]
RECORDER = ROOT / '.github/scripts/record-internal-api-test-seconds.py'
def event(action, test=None, output=None, elapsed=None):
value = {'Action': action, 'Package': 'example/internal/api'}
if test is not None:
value['Test'] = test
if output is not None:
value['Output'] = output
if elapsed is not None:
value['Elapsed'] = elapsed
return json.dumps(value) + '\n'
class APIFailureIndexTest(unittest.TestCase):
def record(self, stream, selected=None, package='example/internal/api'):
with tempfile.TemporaryDirectory() as directory:
seconds = Path(directory) / 'api-2.txt'
command = [sys.executable, str(RECORDER), str(seconds)]
if selected is not None:
selection = Path(directory) / 'api-2.selected'
selection.write_text('\n'.join(selected) + '\n')
command.extend([str(selection), package])
result = subprocess.run(command,
input=stream, text=True, capture_output=True,
timeout=10)
raw = Path(str(seconds) + '.failures.json').read_text()
return result, seconds.read_text(), json.loads(raw), raw
def test_spilled_output_keeps_interleaved_unicode_and_unfinished_order(self):
names = ['TestBroken', 'TestQuiet', 'TestSkipped', 'TestZPending', 'TestAPending']
output = {name: ''.join(f'{i}: λ雪 {name}/private-case\n' for i in range(4096))
for name in names}
stream = ''.join(event('run', name) for name in names)
for i in range(0, max(map(len, output.values())), 1024):
for name in names:
stream += event('output', name + '/private-case', output[name][i:i + 1024])
stream += (event('fail', 'TestBroken', elapsed=.1) +
event('pass', 'TestQuiet', elapsed=.2) +
event('skip', 'TestSkipped', elapsed=.3) + event('fail', elapsed=.4))
result, seconds, index, raw = self.record(stream, names)
self.assertEqual(result.returncode, 1, result.stderr)
expected = output['TestBroken']
for name in ['TestAPending', 'TestZPending']:
expected += f'--- {name} did not finish; its output follows\n' + output[name]
self.assertEqual(result.stdout, expected)
self.assertEqual(seconds, '# package-seconds 0.40\nTestBroken 0.10\n'
'TestQuiet 0.20\nTestSkipped 0.30\n')
self.assertEqual(index['failed_top_level_tests']['names'], ['TestBroken'])
self.assertEqual(index['unfinished_top_level_tests']['names'],
['TestAPending', 'TestZPending'])
self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkipped'])
self.assertNotIn('private-case', raw)
def noisy_record(self, *, interrupted):
# Stream 96 MiB from disk in small events: the recorder, not the test
# driver, must retain all four active roots. No application/Go test or
# excluded crash scenario is involved. A fixed address-space budget
# catches output-sized retention without a timing/RSS threshold.
import resource
names = ['TestBroken', 'TestQuiet', 'TestSkipped', 'TestPending']
chunks = {name: (name + '/private-case λ\r\n').ljust(8192, '.') for name in names}
repeats = 3072
expected = hashlib.sha256()
if interrupted:
for _ in range(repeats):
expected.update(chunks['TestBroken'].encode())
expected.update(b'--- TestPending did not finish; its output follows\n')
for _ in range(repeats):
expected.update(chunks['TestPending'].encode())
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
source, seconds, log = (root / name for name in ['events.jsonl', 'seconds.txt', 'log'])
spool = root / 'spool'
spool.mkdir()
selection = root / 'selected'
selection.write_text('\n'.join(names) + '\n')
with source.open('w', encoding='utf-8', newline='') as stream:
stream.write(''.join(event('run', name) for name in names))
rows = ''.join(event('output', name + '/private-case', chunks[name]) for name in names)
for _ in range(repeats):
stream.write(rows)
stream.write(event('fail' if interrupted else 'pass', names[0], elapsed=.1))
stream.write(event('pass', names[1], elapsed=.2))
stream.write(event('skip', names[2], elapsed=.3))
if not interrupted:
stream.write(event('pass', names[3], elapsed=.4))
stream.write(event('fail' if interrupted else 'pass', elapsed=.5))
def memory_budget():
resource.setrlimit(resource.RLIMIT_AS, (64 * 1024**2, 64 * 1024**2))
with source.open('rb') as stream, log.open('wb') as output:
result = subprocess.run(
[sys.executable, str(RECORDER), str(seconds), str(selection), 'example/internal/api'],
stdin=stream, stdout=output, stderr=subprocess.PIPE, timeout=60,
env={**os.environ, 'TMPDIR': str(spool)}, preexec_fn=memory_budget)
self.assertEqual(result.returncode, int(interrupted), result.stderr.decode())
self.assertNotIn(b'Traceback', result.stderr)
actual = hashlib.sha256()
with log.open('rb') as output:
for block in iter(lambda: output.read(64 * 1024), b''):
actual.update(block)
self.assertEqual(actual.hexdigest(), expected.hexdigest())
raw = Path(str(seconds) + '.failures.json').read_text()
index = json.loads(raw)
self.assertEqual(index['recorder_exit_code'], int(interrupted))
self.assertEqual(index['completed_top_level_test_count'], 3 if interrupted else 4)
self.assertEqual(index['unfinished_top_level_tests']['names'],
['TestPending'] if interrupted else [])
self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkipped'])
self.assertNotIn('private-case', raw)
self.assertEqual(list(spool.iterdir()), [])
@unittest.skipUnless(sys.platform == 'linux', 'Linux CI address-space control')
def test_noisy_passing_and_skipped_roots_fit_fixed_memory(self):
self.noisy_record(interrupted=False)
@unittest.skipUnless(sys.platform == 'linux', 'Linux CI address-space control')
def test_noisy_failed_and_unfinished_roots_fit_fixed_memory_without_log_loss(self):
self.noisy_record(interrupted=True)
def test_pass_preserves_timing_and_quiet_output(self):
result, seconds, index, _ = self.record(
event('run', 'TestGood') + event('output', 'TestGood', 'private output\n') +
event('pass', 'TestGood', elapsed=1.25) + event('pass', elapsed=2))
self.assertEqual(result.returncode, 0)
self.assertEqual(result.stdout, '')
self.assertEqual(seconds, '# package-seconds 2.00\nTestGood 1.25\n')
self.assertEqual(index['failed_top_level_tests'],
{'names': [], 'observed_count': 0, 'omitted_count': 0})
self.assertEqual(index['package_terminal_action'], 'pass')
self.assertEqual(index['package_terminal_count'], 1)
def test_failure_names_root_without_subtest_or_assertion_data(self):
secret = 'customer@example.invalid'
result, seconds, index, raw = self.record(
event('run', 'TestBroken') +
event('output', 'TestBroken/' + secret, secret + '\n') +
event('fail', 'TestBroken/' + secret, elapsed=0.2) +
event('fail', 'TestBroken', elapsed=0.3) + event('fail', elapsed=0.4))
self.assertEqual(result.returncode, 1)
self.assertIn(secret, result.stdout) # Existing diagnostic log stays intact.
self.assertEqual(seconds, '# package-seconds 0.40\nTestBroken 0.30\n')
self.assertEqual(index['failed_top_level_tests']['names'], ['TestBroken'])
self.assertEqual(index['unfinished_top_level_tests']['names'], [])
self.assertEqual(index['package_terminal_action'], 'fail')
self.assertNotIn(secret, raw)
self.assertNotIn('Package', raw)
def test_unfinished_root_is_not_a_completed_failure_or_pass(self):
result, seconds, index, raw = self.record(
event('run', 'TestInterrupted') +
event('output', 'TestInterrupted/private-label', 'private log\n'))
self.assertEqual(result.returncode, 1)
self.assertIn('TestInterrupted did not finish', result.stdout)
self.assertEqual(seconds, '')
self.assertEqual(index['failed_top_level_tests']['names'], [])
self.assertEqual(index['unfinished_top_level_tests']['names'], ['TestInterrupted'])
self.assertIsNone(index['package_terminal_action'])
self.assertEqual(index['package_terminal_count'], 0)
self.assertNotIn('private', raw)
def test_package_only_failure_does_not_invent_a_test_identity(self):
result, _, index, raw = self.record(
event('output', output='private compiler diagnostic\n') + event('fail'))
self.assertEqual(result.returncode, 1)
self.assertIn('compiler diagnostic', result.stdout)
self.assertEqual(index['failed_top_level_tests']['names'], [])
self.assertEqual(index['package_terminal_action'], 'fail')
self.assertNotIn('diagnostic', raw)
def test_non_json_and_empty_streams_remain_unknown(self):
for stream, count in [('', 0), ('private non-json line\n[]\n', 2)]:
with self.subTest(stream=bool(stream)):
result, _, index, raw = self.record(stream)
# Unknown completion cannot become a passing workflow verdict,
# even if a producer happens to return zero.
self.assertEqual(result.returncode, 1)
self.assertEqual(index['non_json_line_count'], count)
self.assertIsNone(index['package_terminal_action'])
self.assertNotIn('private', raw)
def test_unsafe_and_long_source_names_are_counted_but_not_exported(self):
for name in ['TestUnsafe\n::error::private', 'TestUnsafe\x1b[31m',
'TestÜnicode', 'not-a-source-test', 'Test' + 'A' * 257]:
with self.subTest(name=name[:12]):
result, _, index, raw = self.record(event('fail', name))
self.assertEqual(result.returncode, 1)
self.assertEqual(index['failed_top_level_tests'],
{'names': [], 'observed_count': 1, 'omitted_count': 1})
self.assertNotIn(name, raw)
def test_lists_are_deduplicated_sorted_and_bounded_separately(self):
stream = ''.join(event('fail', f'TestFailed{i:04d}') for i in range(300, -1, -1))
stream += event('fail', 'TestFailed0000')
stream += ''.join(event('run', f'TestPending{i:04d}') for i in range(300, -1, -1))
result, _, index, raw = self.record(stream)
self.assertEqual(result.returncode, 1)
for field in ['failed_top_level_tests', 'unfinished_top_level_tests']:
value = index[field]
self.assertEqual(len(value['names']), 256)
self.assertEqual(value['observed_count'], 301)
self.assertEqual(value['omitted_count'], 45)
self.assertEqual(value['names'], sorted(set(value['names'])))
self.assertLess(len(raw.encode()), 150_000)
def test_package_actions_preserve_contrary_observations_without_green_claim(self):
result, _, index, _ = self.record(event('fail') + event('pass'))
self.assertEqual(result.returncode, 1)
self.assertEqual(index['package_terminal_action'], 'pass')
self.assertEqual(index['package_terminal_count'], 2)
self.assertNotIn('success', index)
self.assertEqual(index['recorder_exit_code'], 1)
def test_skip_and_supported_source_name_forms(self):
stream = ''.join(event('fail', name) for name in
['Test', 'TestUnder_score123', 'Example', 'ExampleThing', 'FuzzThing'])
result, _, index, _ = self.record(stream + event('skip'))
self.assertEqual(result.returncode, 1)
self.assertEqual(index['failed_top_level_tests']['observed_count'], 5)
self.assertEqual(index['failed_top_level_tests']['omitted_count'], 0)
self.assertEqual(index['package_terminal_action'], 'skip')
def test_sidecar_is_ignored_by_the_existing_timing_refresh(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
seconds = root / 'api-2.txt'
(root / 'api-2.selected').write_text('TestTimed\nTestSelectionIsNotATiming\n')
result = subprocess.run([sys.executable, str(RECORDER), str(seconds)],
input=event('run', 'TestTimed') +
event('pass', 'TestTimed', elapsed=1.25) +
event('pass', elapsed=2),
text=True, capture_output=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
weights = root / 'weights.out'
result = subprocess.run(
[sys.executable, str(ROOT / '.github/scripts/refresh-internal-api-test-seconds.py'),
'--dir', str(root), '--output', str(weights)],
text=True, capture_output=True, timeout=10)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('TestTimed 1.25', weights.read_text())
self.assertNotIn('failures.json', result.stdout)
def test_pipefail_preserves_an_upstream_failure_with_no_json(self):
with tempfile.TemporaryDirectory() as directory:
seconds = Path(directory) / 'api-2.txt'
result = subprocess.run(
['bash', '-c', 'set -o pipefail; (printf "compile diagnostic\\n"; exit 17) | '
'python3 "$1" "$2"', 'recorder-proof', str(RECORDER), str(seconds)],
text=True, capture_output=True, timeout=10)
# Both producer and recorder fail; pipefail returns the last
# failing stage rather than manufacturing success.
self.assertEqual(result.returncode, 1)
index = json.loads(Path(str(seconds) + '.failures.json').read_text())
self.assertIsNone(index['package_terminal_action'])
self.assertEqual(index['recorder_exit_code'], 1)
self.assertEqual(index['non_json_line_count'], 1)
def test_existing_upload_collects_sidecar_on_a_failed_shard(self):
workflow = (ROOT / '.github/workflows/build-and-test.yml').read_text()
job = workflow.split('\n backend-api:\n', 1)[1].split('\n backend-api-verdict:', 1)[0]
self.assertIn('| python3 .github/scripts/record-internal-api-test-seconds.py', job)
self.assertIn('if: always() && needs.changes.outputs.code ==', job)
self.assertIn('path: ${{ runner.temp }}/internal-api-test-seconds/', job)
self.assertIn('name: internal-api-test-seconds-${{ matrix.index }}', job)
self.assertIn('set -euo pipefail', job)
parsed_job = yaml.safe_load(workflow)['jobs']['backend-api']
self.assert_api_validation_remains_fatal(parsed_job)
self.assertIn('printf \'%s\\n\' "$selected" > "$timings/api-${API_SHARD_INDEX}.selected"', job)
self.assertIn('package=$(go list ./internal/api)', job)
self.assertIn('"$timings/api-${API_SHARD_INDEX}.selected" "$package"', job)
def assert_api_validation_remains_fatal(self, job):
# Optional object transport is not a validation result. Keep job,
# compile admission, runtime pipeline and sidecar upload failures fatal;
# allow only the exact pinned, read-only cache restore, not its name.
self.assertNotIn('continue-on-error', job)
prefix = ("backend-race-objects-v1-${{ runner.os }}-${{ runner.arch }}-ubuntu24-"
"${{ steps.go-setup.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }}-")
optional_restore = {
'name': 'Restore backend race build objects',
'if': "needs.changes.outputs.code == 'true'",
'id': 'race-cache',
'continue-on-error': True,
'uses': 'actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9',
'with': {'path': '${{ steps.go-build-cache.outputs.path }}',
'key': prefix + '${{ github.sha }}', 'restore-keys': prefix},
}
optional_steps = [step for step in job['steps'] if 'continue-on-error' in step]
self.assertLessEqual(len(optional_steps), 1)
for step in optional_steps:
self.assertEqual(step, optional_restore)
def test_optional_cache_cannot_make_validation_or_sidecar_failure_nonfatal(self):
job = yaml.safe_load((ROOT / '.github/workflows/build-and-test.yml').read_text())['jobs']['backend-api']
self.assert_api_validation_remains_fatal(job)
no_cache = copy.deepcopy(job)
no_cache['steps'] = [step for step in no_cache['steps'] if step.get('id') != 'race-cache']
self.assert_api_validation_remains_fatal(no_cache)
for name in ('Require compiled backend tests',
'Go unit tests (internal/api shard ${{ matrix.index }})',
'Upload internal/api test timings'):
with self.subTest(masked_step=name):
changed = copy.deepcopy(job)
step = next(step for step in changed['steps'] if step['name'] == name)
step['continue-on-error'] = True
with self.assertRaises(AssertionError):
self.assert_api_validation_remains_fatal(changed)
changed = copy.deepcopy(job)
changed['continue-on-error'] = True
with self.assertRaises(AssertionError):
self.assert_api_validation_remains_fatal(changed)
# A lookalike cache step must not smuggle arbitrary execution, artifact
# transport, paths or conditions through the narrowly optional boundary.
for field, value in (
('uses', 'actions/cache/restore@main'),
('uses', 'actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9'),
('run', 'exit 0'), ('if', 'always()'),
('with', {'path': '${{ github.workspace }}'})):
with self.subTest(cache_drift=(field, value)):
changed = copy.deepcopy(job)
step = next(step for step in changed['steps'] if step.get('id') == 'race-cache')
step[field] = value
with self.assertRaises(AssertionError):
self.assert_api_validation_remains_fatal(changed)
def test_selected_pass_and_explicit_skip_complete_coverage(self):
stream = (event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2) +
event('run', 'TestNative') + event('skip', 'TestNative', elapsed=0) +
event('pass', elapsed=0.3))
result, _, index, _ = self.record(stream, ['TestGood', 'TestNative'])
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(index['selected_test_count'], 2)
self.assertEqual(index['completed_top_level_test_count'], 2)
self.assertEqual(index['missing_selected_tests']['observed_count'], 0)
self.assertEqual(index['unexpected_top_level_tests']['observed_count'], 0)
self.assertIn('skipped_top_level_tests', index)
self.assertEqual(index['skipped_top_level_tests'],
{'names': ['TestNative'], 'observed_count': 1, 'omitted_count': 0})
self.assertEqual(index['top_level_tests_with_skipped_subtests']['observed_count'], 0)
self.assertEqual(index['skipped_subtest_event_count'], 0)
def test_skipped_subtests_do_not_disappear_under_a_passing_parent(self):
private = 'customer@example.invalid/secret-case'
stream = (event('run', 'TestPartial') +
event('run', 'TestPartial/' + private) +
event('output', 'TestPartial/' + private, 'private skip reason\n') +
event('skip', 'TestPartial/' + private) +
event('run', 'TestPartial/another-private-case') +
event('skip', 'TestPartial/another-private-case') +
event('pass', 'TestPartial', elapsed=0.5) + event('pass', elapsed=0.6))
result, seconds, index, raw = self.record(stream, ['TestPartial'])
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(result.stdout, '')
self.assertEqual(seconds, '# package-seconds 0.60\nTestPartial 0.50\n')
self.assertIn('top_level_tests_with_skipped_subtests', index)
self.assertEqual(index['top_level_tests_with_skipped_subtests'],
{'names': ['TestPartial'], 'observed_count': 1, 'omitted_count': 0})
self.assertEqual(index['skipped_top_level_tests']['observed_count'], 0)
self.assertEqual(index['skipped_subtest_event_count'], 2)
for hidden in [private, 'another-private-case', 'private skip reason']:
self.assertNotIn(hidden, raw)
def test_skip_observations_survive_failure_and_missing_completion(self):
partial = (event('run', 'TestPartial') +
event('skip', 'TestPartial/private-case') +
event('run', 'TestUnavailable') + event('skip', 'TestUnavailable'))
for terminal in ['', event('fail', 'TestPartial') + event('fail')]:
with self.subTest(completed=bool(terminal)):
result, _, index, raw = self.record(
partial + terminal, ['TestPartial', 'TestUnavailable'])
self.assertEqual(result.returncode, 1)
self.assertIn('skipped_top_level_tests', index)
self.assertEqual(index['skipped_top_level_tests']['names'], ['TestUnavailable'])
self.assertEqual(index['top_level_tests_with_skipped_subtests']['names'],
['TestPartial'])
self.assertEqual(index['skipped_subtest_event_count'], 1)
self.assertNotIn('private-case', raw)
field = 'failed_top_level_tests' if terminal else 'unfinished_top_level_tests'
self.assertEqual(index[field]['names'], ['TestPartial'])
def test_skip_identifiers_are_bounded_and_private_labels_are_not_exported(self):
names = [f'TestSkip{i:04d}' for i in range(300, -1, -1)]
names += ['TestÜnicode', 'Test' + 'A' * 257, 'not-a-source-test']
stream = ''.join(event('run', name) + event('skip', name + '/private-child') +
event('skip', name) for name in names) + event('pass')
result, _, index, raw = self.record(stream)
self.assertEqual(result.returncode, 0, result.stderr)
for field in ['skipped_top_level_tests', 'top_level_tests_with_skipped_subtests']:
self.assertIn(field, index)
observed = index[field]
self.assertEqual(observed['observed_count'], 304)
self.assertEqual(len(observed['names']), 256)
self.assertEqual(observed['omitted_count'], 48)
self.assertEqual(observed['names'], sorted(set(observed['names'])))
self.assertEqual(index['skipped_subtest_event_count'], 304)
for hidden in ['private-child', 'Ünicode', 'not-a-source-test', 'A' * 257]:
self.assertNotIn(hidden, raw)
self.assertLess(len(raw.encode()), 150_000)
def test_successful_producer_cannot_substitute_incomplete_execution(self):
good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)
for stream in ['', event('pass'), good, 'malformed\n' + good + event('pass'),
good + event('pass') + event('pass'),
event('pass', 'TestGood') + event('pass'),
good + good + event('pass'), good + event('skip')]:
with self.subTest(stream=stream):
result, _, index, _ = self.record(stream, ['TestGood'])
self.assertEqual(result.returncode, 1)
self.assertEqual(index['recorder_exit_code'], 1)
self.assertIn('no passing verdict', result.stderr)
def test_selected_tests_must_not_be_dropped_or_replaced(self):
good = event('run', 'TestGood') + event('pass', 'TestGood', elapsed=0.2)
result, _, index, _ = self.record(good + event('pass'), ['TestGood', 'TestMissing'])
self.assertEqual(result.returncode, 1)
self.assertEqual(index['missing_selected_tests']['names'], ['TestMissing'])
other = event('run', 'TestOther') + event('pass', 'TestOther', elapsed=0.2)
result, _, index, _ = self.record(other + event('pass'), ['TestGood'])
self.assertEqual(result.returncode, 1)
self.assertEqual(index['missing_selected_tests']['names'], ['TestGood'])
self.assertEqual(index['unexpected_top_level_tests']['names'], ['TestOther'])
def test_unfinished_selected_test_retains_both_coverage_and_original_log(self):
stream = event('run', 'TestHung') + event('output', 'TestHung/private', 'private log\n')
result, _, index, raw = self.record(stream, ['TestHung'])
self.assertEqual(result.returncode, 1)
self.assertIn('private log', result.stdout)
self.assertEqual(index['unfinished_top_level_tests']['names'], ['TestHung'])
self.assertEqual(index['missing_selected_tests']['names'], ['TestHung'])
self.assertNotIn('private', raw)
def test_wrong_or_multiple_packages_cannot_satisfy_selection(self):
good = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
for stream, package in [(good, 'other/private-package'),
(good.replace('example/internal/api', 'other/private-package', 1),
'example/internal/api')]:
result, _, index, raw = self.record(stream, ['TestGood'], package)
self.assertEqual(result.returncode, 1)
self.assertGreater(index['unexpected_package_event_count'], 0)
self.assertNotIn('private-package', raw)
def test_invalid_event_shapes_are_non_passing_not_recorder_crashes(self):
for changed in [{'Action': 'invented'}, {'Package': None}, {'Test': []}, {'Output': 1}]:
value = {'Action': 'output', 'Package': 'example/internal/api'}
value.update(changed)
stream = (json.dumps(value) + '\n' + event('run', 'TestGood') +
event('pass', 'TestGood') + event('pass'))
result, _, index, _ = self.record(stream, ['TestGood'])
self.assertEqual(result.returncode, 1)
self.assertGreater(index['invalid_event_count'], 0)
self.assertNotIn('Traceback', result.stderr)
def test_go_build_diagnostics_remain_visible_without_test_package_identity(self):
good = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
for action, code in [('build-output', 0), ('build-fail', 1)]:
build = json.dumps({'Action': action, 'ImportPath': 'private/dependency',
'Output': 'retained build diagnostic\n'}) + '\n'
result, _, index, raw = self.record(build + good, ['TestGood'])
self.assertEqual(result.returncode, code)
self.assertIn('retained build diagnostic', result.stdout)
self.assertEqual(index['package_identity_count'], 1)
self.assertNotIn('private/dependency', raw)
self.assertNotIn('diagnostic', raw)
def test_coverage_names_stay_bounded_without_rejecting_unicode_source(self):
missing = [f'TestMissing{i:04d}' for i in range(301)]
missing.extend(['TestÜnicode', 'Test' + 'A' * 257])
stream = event('run', 'TestGood') + event('pass', 'TestGood') + event('pass')
result, _, index, raw = self.record(stream, ['TestGood'] + missing)
self.assertEqual(result.returncode, 1)
observation = index['missing_selected_tests']
self.assertEqual(observation['observed_count'], 303)
self.assertEqual(len(observation['names']), 256)
self.assertEqual(observation['omitted_count'], 47)
self.assertNotIn('Ünicode', raw)
result, _, _, _ = self.record(
event('run', 'TestÜnicode') + event('pass', 'TestÜnicode') + event('pass'),
['TestÜnicode'])
self.assertEqual(result.returncode, 0)
def test_invalid_selection_is_rejected_before_observation(self):
with tempfile.TemporaryDirectory() as directory:
seconds = Path(directory) / 'api.txt'
selected = Path(directory) / 'api.selected'
for names in ['', 'TestA\nTestA\n', 'TestA/subtest\n', 'TestBad-name\n']:
selected.write_text(names)
result = subprocess.run([sys.executable, str(RECORDER), str(seconds),
str(selected), 'example/internal/api'],
input='', text=True, capture_output=True, timeout=10)
self.assertEqual(result.returncode, 2)
self.assertFalse(seconds.exists())
selected.unlink()
result = subprocess.run([sys.executable, str(RECORDER), str(seconds),
str(selected), 'example/internal/api'],
input='', text=True, capture_output=True, timeout=10)
self.assertEqual(result.returncode, 2)
self.assertIn('unavailable', result.stderr)
@unittest.skipUnless(shutil.which('go'), 'real Go JSON requires the declared compiler')
def test_real_race_json_pass_skip_fuzz_example_failure_and_missing_coverage(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
package = 'example.invalid/shardfixture'
(root / 'go.mod').write_text(f'module {package}\n\ngo 1.24\n')
(root / 'fixture_test.go').write_text('''package shardfixture
import ("fmt"; "testing")
type Fixture struct{}
func TestGood(t *testing.T) { t.Log("passing fixture output") }
func TestSkip(t *testing.T) { t.Skip("explicit native fixture exclusion") }
func TestPartial(t *testing.T) {
t.Run("working", func(t *testing.T) {})
t.Run("private-child", func(t *testing.T) { t.Skip("private native fixture reason") })
}
func TestFailure(t *testing.T) { t.Fatal("retained fixture failure") }
func FuzzSeed(f *testing.F) { f.Add(1); f.Fuzz(func(t *testing.T, n int) {}) }
func ExampleFixture() { fmt.Println("fixture")
// Output: fixture
}
''')
env = {**os.environ, 'GOWORK': 'off', 'GOPROXY': 'off', 'GOTOOLCHAIN': 'local'}
listing = subprocess.run(['go', 'test', '-race', '-list', '.', '.'],
cwd=root, env=env, text=True, capture_output=True, timeout=120)
self.assertEqual(listing.returncode, 0, listing.stderr)
names = [name for name in listing.stdout.splitlines()
if name.startswith(('Test', 'Fuzz', 'Example'))]
selected = [name for name in names if name != 'TestFailure']
self.assertEqual(set(selected), {'TestGood', 'TestSkip', 'TestPartial', 'FuzzSeed', 'ExampleFixture'})
command = ['go', 'test', '-race', '-count=1', '-timeout', '30s', '-json']
# Exercise the same shorter-side -skip path used by large CI shards.
passing = subprocess.run(command + ['-skip', '^TestFailure$', '.'],
cwd=root, env=env, text=True, capture_output=True, timeout=120)
self.assertEqual(passing.returncode, 0, passing.stderr)
result, _, index, _ = self.record(passing.stdout, selected, package)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(index['completed_top_level_test_count'], 5)
self.assertIn('skipped_top_level_tests', index)
self.assertEqual(index['skipped_top_level_tests']['names'], ['TestSkip'])
self.assertEqual(index['top_level_tests_with_skipped_subtests']['names'], ['TestPartial'])
self.assertEqual(index['skipped_subtest_event_count'], 1)
self.assertNotIn('private-child', json.dumps(index))
self.assertNotIn('private native fixture reason', json.dumps(index))
self.assertNotIn('passing fixture output', result.stdout)
# Real Go exits zero while selecting fewer tests than the source
# list expects. Neither pipefail nor a package pass detects that.
omitted = subprocess.run(command + ['-run', '^TestGood$', '.'],
cwd=root, env=env, text=True, capture_output=True, timeout=120)
self.assertEqual(omitted.returncode, 0, omitted.stderr)
result, _, index, _ = self.record(omitted.stdout, selected, package)
self.assertEqual(result.returncode, 1)
self.assertEqual(index['missing_selected_tests']['observed_count'], 4)
# Completed tests without the final package event are not a pass.
events = [json.loads(line) for line in passing.stdout.splitlines()]
truncated = ''.join(json.dumps(value) + '\n' for value in events
if not (value.get('Action') == 'pass' and 'Test' not in value))
result, _, index, _ = self.record(truncated, selected, package)
self.assertEqual(result.returncode, 1)
self.assertIsNone(index['package_terminal_action'])
failing = subprocess.run(command + ['-run', '^TestFailure$', '.'],
cwd=root, env=env, text=True, capture_output=True, timeout=120)
self.assertEqual(failing.returncode, 1, failing.stderr)
result, _, index, _ = self.record(failing.stdout, ['TestFailure'], package)
self.assertEqual(result.returncode, 1)
self.assertIn('retained fixture failure', result.stdout)
self.assertEqual(index['failed_top_level_tests']['names'], ['TestFailure'])
self.assertEqual(index['missing_selected_tests']['observed_count'], 0)
print('Real Go race JSON: complete pass/skip/fuzz/example accepted; '
'producer-zero omission, truncated completion and actual failure rejected.')
if __name__ == '__main__':
unittest.main()