unyo-web/scripts/generate_extension_health_report.py

#!/usr/bin/env python3
"""
Generate a Docusaurus documentation page from an Aniyomi extension batch test JSON report.

Usage:
    python3 scripts/generate_extension_health_report.py <path/to/report.json>

The script reads the report JSON and writes a formatted Markdown file to:
    website/docs/extensions-compatibility.md

Example:
    python3 scripts/generate_extension_health_report.py reports/2026-05-04_01-59-09/report.json
"""

import json
import sys
import os
from collections import Counter, defaultdict
import statistics


def fmt_duration(ms):
    """Format a duration in milliseconds to a human-readable string."""
    if ms < 0:
        return '—'
    if ms < 1000:
        return f"{ms}ms"
    return f"{ms / 1000:.1f}s"


def extract_short_error(error_msg):
    """Extract a concise error description from a full stack trace."""
    if not error_msg:
        return '—'
    lines = error_msg.strip().split('\n')
    for line in lines:
        line = line.strip()
        if 'UnknownHostException' in line:
            return 'UnknownHostException'
        if 'HttpException: HTTP error' in line:
            return line.split('HttpException: ')[1][:30]
        if 'ConnectException: Connection refused' in line:
            return 'Connection refused'
        if 'ConnectException' in line:
            return 'ConnectException'
        if 'CloudflareBypassException' in line or 'Failed to bypass Cloudflare' in line:
            return 'Cloudflare bypass failed'
        if 'SSL' in line or 'Certificate' in line:
            return 'SSL/TLS error'
        if 'SocketTimeoutException' in line or 'timeout' in line.lower():
            return 'Timeout'
        if 'IllegalArgumentException' in line:
            if 'extension settings' in line.lower():
                return 'Missing settings'
            return 'IllegalArgumentException'
        if 'NullPointerException' in line:
            return 'NullPointerException'
        if 'NoSuchMethodError' in line:
            return 'NoSuchMethodError'
        if 'AbstractMethodError' in line:
            return 'AbstractMethodError'
        if 'Episode list not found' in line:
            return 'Episode list not found'
        if 'JSON token' in line or 'JSON input' in line:
            return 'JSON parsing error'
        if 'IndexOutOfBoundsException' in line:
            return 'IndexOutOfBoundsException'
        if 'IOException' in line:
            return 'IOException'
    # Fallback: first meaningful line
    for line in lines:
        if line and not line.startswith('at ') and not line.startswith('\tat '):
            return line[:60]
    return 'See error details'


def generate_report(json_path, output_path):
    """Generate the Markdown report from a JSON batch test result."""
    with open(json_path) as f:
        data = json.load(f)

    meta = data['meta']
    summary = data['summary']
    results = data['results']

    # Repository stats
    repo_pass = Counter(r['repoId'] for r in results if r['overallStatus'] == 'PASS')
    repo_fail = Counter(r['repoId'] for r in results if r['overallStatus'] == 'FAIL')

    # Stage stats
    stage_stats = {}
    for stage in ['load', 'search', 'episodes', 'videos']:
        passes = sum(1 for r in results if r[stage]['status'] == 'PASS')
        fails = sum(1 for r in results if r[stage]['status'] == 'FAIL')
        pendings = sum(1 for r in results if r[stage]['status'] == 'PENDING')
        durations = [r[stage]['durationMs'] for r in results if r[stage]['durationMs'] >= 0]
        stage_stats[stage] = {
            'pass': passes,
            'fail': fails,
            'pending': pendings,
            'median': statistics.median(durations) if durations else 0,
            'max': max(durations) if durations else 0,
            'min': min(durations) if durations else 0,
        }

    # Working extensions (all that passed every stage)
    working_extensions = []
    for r in results:
        if r['overallStatus'] == 'PASS':
            total = sum(r[stage]['durationMs'] for stage in ['load', 'search', 'episodes', 'videos'])
            working_extensions.append({
                'name': r['name'],
                'repo': r['repoId'],
                'lang': r['lang'],
                'total': total,
                'load': r['load']['durationMs'],
                'search': r['search']['durationMs'],
                'episodes': r['episodes']['durationMs'],
                'videos': r['videos']['durationMs'],
                'search_results': r['searchResultCount'],
                'episode_results': r['episodeResultCount'],
                'video_results': r['videoResultCount'],
            })
    working_extensions.sort(key=lambda x: x['total'], reverse=True)

    # Failures grouped by stage
    failures_by_stage = defaultdict(list)
    for r in results:
        if r['overallStatus'] == 'FAIL':
            failures_by_stage[r['failedAtStage']].append(r)

    # Build markdown
    md = f"""---
title: Extension Compatibility
description: Automated batch test results for Aniyomi extensions ({meta['timestamp'][:10]})
---

# Extension Compatibility

This report presents the results of an automated batch test run against **{summary['total']}** Aniyomi extensions across the **yuzono** and **kohiden** repositories. The test validates each extension through four stages: loading, searching, episode listing, and video extraction.

:::info Test Methodology
Each extension is tested with the query **`{meta['query']}`** using **{meta['concurrency']}** concurrent threads. A failure at any stage prevents subsequent stages from running. Results depend on network conditions, geo-location, and the current state of third-party sites.
:::

## Test Configuration

| Parameter | Value |
|-----------|-------|
| **Timestamp** | `{meta['timestamp']}` |
| **Query** | `{meta['query']}` |
| **Concurrency** | {meta['concurrency']} threads |
| **Duration** | {fmt_duration(meta['totalDurationMs'])} |
| **Repositories** | {', '.join(meta['repos'])} |
| **Total Extensions** | {summary['total']} |

## Overall Summary

| Metric | Count | Percentage |
|--------|-------|------------|
| **Pass** | {summary['pass']} | {summary['pass'] / summary['total'] * 100:.1f}% |
| **Fail** | {summary['fail']} | {summary['fail'] / summary['total'] * 100:.1f}% |
| **Skip** | {summary['skip']} | {summary['skip'] / summary['total'] * 100:.1f}% |
| **Total** | {summary['total']} | 100.0% |

### By Repository

| Repository | Total | Pass | Fail | Pass Rate |
|------------|-------|------|------|-----------|
| yuzono | {repo_pass['yuzono'] + repo_fail['yuzono']} | {repo_pass['yuzono']} | {repo_fail['yuzono']} | {repo_pass['yuzono'] / (repo_pass['yuzono'] + repo_fail['yuzono']) * 100:.1f}% |
| kohiden | {repo_pass['kohiden'] + repo_fail['kohiden']} | {repo_pass['kohiden']} | {repo_fail['kohiden']} | {repo_pass['kohiden'] / (repo_pass['kohiden'] + repo_fail['kohiden']) * 100:.1f}% |

## Working Extensions

The {len(working_extensions)} extensions that passed all four stages, sorted by total test duration:

| Extension | Repository | Language | Load | Search | Episodes | Videos | Total | Results (S/E/V) |
|-----------|------------|----------|------|--------|----------|--------|-------|-----------------|
"""
    for ext in working_extensions:
        md += (f"| {ext['name']} | {ext['repo']} | {ext['lang']} | {fmt_duration(ext['load'])} | "
               f"{fmt_duration(ext['search'])} | {fmt_duration(ext['episodes'])} | "
               f"{fmt_duration(ext['videos'])} | {fmt_duration(ext['total'])} | "
               f"{ext['search_results']}/{ext['episode_results']}/{ext['video_results']} |\n")

    md += f"""
## Stage Breakdown

Each extension progresses through four stages. The tables below show how many extensions passed, failed, or never reached each stage.

| Stage | Pass | Fail | Pending | Median Time | Max Time |
|-------|------|------|---------|-------------|----------|
| Load | {stage_stats['load']['pass']} | {stage_stats['load']['fail']} | {stage_stats['load']['pending']} | {fmt_duration(int(stage_stats['load']['median']))} | {fmt_duration(stage_stats['load']['max'])} |
| Search | {stage_stats['search']['pass']} | {stage_stats['search']['fail']} | {stage_stats['search']['pending']} | {fmt_duration(int(stage_stats['search']['median']))} | {fmt_duration(stage_stats['search']['max'])} |
| Episodes | {stage_stats['episodes']['pass']} | {stage_stats['episodes']['fail']} | {stage_stats['episodes']['pending']} | {fmt_duration(int(stage_stats['episodes']['median']))} | {fmt_duration(stage_stats['episodes']['max'])} |
| Videos | {stage_stats['videos']['pass']} | {stage_stats['videos']['fail']} | {stage_stats['videos']['pending']} | {fmt_duration(int(stage_stats['videos']['median']))} | {fmt_duration(stage_stats['videos']['max'])} |

### Failure Distribution

| Stage | Failures | Percentage of Total |
|-------|----------|---------------------|
| Load | {summary['failAtLoad']} | {summary['failAtLoad'] / summary['total'] * 100:.1f}% |
| Search | {summary['failAtSearch']} | {summary['failAtSearch'] / summary['total'] * 100:.1f}% |
| Episodes | {summary['failAtEpisodes']} | {summary['failAtEpisodes'] / summary['total'] * 100:.1f}% |
| Videos | {summary['failAtVideos']} | {summary['failAtVideos'] / summary['total'] * 100:.1f}% |

## Failures by Stage

:::danger Load Failures ({len(failures_by_stage.get('load', []))})
Extensions that failed to initialize.
:::

| Extension | Repository | Error |
|-----------|------------|-------|
"""
    for r in failures_by_stage.get('load', []):
        md += f"| {r['name']} | {r['repoId']} | {extract_short_error(r.get('errorMessage', ''))} |\n"

    md += f"""
:::danger Search Failures ({len(failures_by_stage.get('search', []))})
Extensions that loaded successfully but failed to return search results.
:::

| Extension | Repository | Error |
|-----------|------------|-------|
"""
    for r in failures_by_stage.get('search', [])[:50]:
        md += f"| {r['name']} | {r['repoId']} | {extract_short_error(r.get('errorMessage', ''))} |\n"
    if len(failures_by_stage.get('search', [])) > 50:
        md += f"| ... | ... | *and {len(failures_by_stage['search']) - 50} more* |\n"

    md += f"""
:::caution Episode Failures ({len(failures_by_stage.get('episodes', []))})
Extensions that passed search but failed to retrieve episode lists.
:::

| Extension | Repository | Error |
|-----------|------------|-------|
"""
    for r in failures_by_stage.get('episodes', []):
        md += f"| {r['name']} | {r['repoId']} | {extract_short_error(r.get('errorMessage', ''))} |\n"

    md += f"""
:::caution Video Failures ({len(failures_by_stage.get('videos', []))})
Extensions that passed episodes but failed to extract video URLs.
:::

| Extension | Repository | Error |
|-----------|------------|-------|
"""
    for r in failures_by_stage.get('videos', []):
        md += f"| {r['name']} | {r['repoId']} | {extract_short_error(r.get('errorMessage', ''))} |\n"

    md += """
## Notes

- **Test Environment**: Results are highly dependent on the test runner's network conditions, DNS resolution, and geographic location.
- **Cloudflare**: Many failures are due to Cloudflare protection. These extensions may work fine in the actual Unyo app with proper WebView support.
- **Configuration Required**: Some extensions (e.g., Google Drive, Jellyfin) require user settings that were not provided during the automated test.
- **Site Changes**: Extensions relying on third-party websites may break when those sites change their layout or domain.
- **API Mismatches**: Errors like `NoSuchMethodError` or `AbstractMethodError` indicate the extension was built against a different version of the Aniyomi/Tachiyomi API than what the bridge provides.

*Report generated automatically from batch test results.*
"""

    os.makedirs(os.path.dirname(output_path), exist_ok=True)
    with open(output_path, 'w') as f:
        f.write(md)

    print(f"Generated {output_path}")
    print(f"Total lines: {len(md.splitlines())}")


def main():
    if len(sys.argv) < 2:
        print("Usage: python3 generate_extension_health_report.py <path/to/report.json>")
        sys.exit(1)

    json_path = sys.argv[1]
    if not os.path.exists(json_path):
        print(f"Error: File not found: {json_path}")
        sys.exit(1)

    # Default output path: website/docs/extensions-compatibility.md
    output_path = os.path.join('website', 'docs', 'extensions-compatibility.md')
    generate_report(json_path, output_path)


if __name__ == "__main__":
    main()