Unyo/unyo-core
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Attempt to improve WebView

Browse Source
This commit is contained in:
Kevin Rodrigues Borges
2026-05-11 03:30:03 +01:00
parent e336275363
commit 923ae5fbbc
8 changed files with 290 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
androidcompat/pom.xml
View File

@@ -190,6 +190,12 @@
<exclude name="resources.arsc"/>
<exclude name="org/json/**"/>
<exclude name="org/apache/http/**"/>
<!-- Strip JDK duplicates (android.jar shadows the JVM bootstrap classes) -->
<exclude name="java/**"/>
<exclude name="javax/**"/>
<exclude name="org/w3c/**"/>
<exclude name="org/xml/**"/>
<exclude name="dalvik/**"/>
</patternset>
</unzip>
<echo message="android.jar unpacked successfully"/>

85
androidcompat/src/main/java/android/webkit/WebView.java
View File

@@ -45,13 +45,85 @@ public class WebView {
webClient.getOptions().setUseInsecureSSL(true); // Allow SSL errors
// Set timeouts
webClient.getOptions().setTimeout(30000); // 30 seconds
webClient.getOptions().setTimeout(20000); // 20 seconds
// Disable downloading of images and CSS for performance
webClient.getOptions().setDownloadImages(false);
// Set cookie manager integration
webClient.getCookieManager().setCookiesEnabled(true);
// Suppress harmless "Obsolete content type" warnings for application/javascript.
// Cloudflare serves challenge scripts with the legacy MIME type; HtmlUnit
// processes them correctly but logs a warning that clutters output.
webClient.setIncorrectnessListener(new org.htmlunit.IncorrectnessListener() {
@Override
public void notify(String message, Object origin) {
if (message != null && message.contains("Obsolete content type encountered")) {
return; // silently ignore
}
// Delegate everything else to the default listener so we still
// see genuine problems.
new org.htmlunit.IncorrectnessListenerImpl().notify(message, origin);
}
});
// Install a script pre-processor that neutralises Web Workers.
// Cloudflare's challenge sometimes spins up Workers; HtmlUnit's
// DedicatedWorkerGlobalScope initialisation is buggy (throws
// IllegalStateException on setImmunePrototypeProperty) which causes
// the challenge to retry every ~30 s. By returning a lightweight
// mock that immediately dispatches an 'error' event, the script
// gets a clean failure signal and can fall back to main-thread
// execution far quicker.
webClient.setScriptPreProcessor(new org.htmlunit.ScriptPreProcessor() {
@Override
public String preProcess(
org.htmlunit.html.HtmlPage page,
String sourceCode,
String sourceName,
int lineNumber,
org.htmlunit.html.HtmlElement htmlElement) {
// Only inject once per page (guard inside the JS itself).
if (sourceCode == null || sourceCode.trim().isEmpty()) {
return sourceCode;
}
String guard = "if(window.__workerMockInstalled){}";
if (sourceCode.contains(guard)) {
return sourceCode; // already injected
}
String workerMock =
"(function(){" +
" if(window.__workerMockInstalled)return;" +
" window.__workerMockInstalled=true;" +
" var RealWorker=window.Worker;" +
" window.Worker=function Worker(url){" +
" var self=this;" +
" this.postMessage=function(){};" +
" this.terminate=function(){};" +
" this.addEventListener=function(type,listener){" +
" if(type==='error'){" +
" setTimeout(function(){" +
" if(self.onerror){" +
" try{self.onerror(new ErrorEvent('error',{message:'Worker blocked'}));}catch(e){}" +
" }" +
" },0);" +
" }" +
" };" +
" this.removeEventListener=function(){};" +
" setTimeout(function(){" +
" if(self.onerror){" +
" try{self.onerror(new ErrorEvent('error',{message:'Worker blocked'}));}catch(e){}" +
" }" +
" },10);" +
" return this;" +
" };" +
" if(RealWorker)window.Worker.prototype=RealWorker.prototype;" +
"})();";
return workerMock + sourceCode;
}
});
}
/**
@@ -112,9 +184,12 @@ public class WebView {
Page page = webClient.getPage(webRequest);
int statusCode = page.getWebResponse().getStatusCode();
// Wait for JavaScript to execute (only for HTML pages)
// Wait for JavaScript to execute (only for HTML pages).
// Cloudflare's challenge is CPU-heavy in Rhino; give it a bit of
// headroom but keep it under the overall latch timeout so we don't
// cut off the callback that delivers the cf_clearance cookie.
if (page instanceof HtmlPage) {
webClient.waitForBackgroundJavaScript(10000); // Wait up to 10 seconds
webClient.waitForBackgroundJavaScript(18000); // Wait up to 18 seconds
}
// Track the actual final URL after any redirects
@@ -140,8 +215,8 @@ public class WebView {
page.getWebResponse().getContentType() != null
? page.getWebResponse().getContentType()
: "text/html",
page.getWebResponse().getContentCharsetOrNull() != null
? page.getWebResponse().getContentCharsetOrNull().name()
page.getWebResponse().getHeaderContentCharset() != null
? page.getWebResponse().getHeaderContentCharset().name()
: "UTF-8",
statusCode,
reasonPhrase,

69
aniyomi-bridge/pom.xml
View File

@@ -181,7 +181,7 @@
</goals>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<minimizeJar>true</minimizeJar>
<minimizeJar>false</minimizeJar>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
@@ -192,14 +192,41 @@
<filter>
<artifact>*:*</artifact>
<excludes>
<!-- Signing artifacts -->
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<!-- JPMS -->
<exclude>module-info.class</exclude>
<exclude>META-INF/versions/*/module-info.class</exclude>
<!-- Merged manifests from dependencies -->
<exclude>META-INF/MANIFEST.MF</exclude>
<!-- License/notice files (many variants) -->
<exclude>META-INF/LICENSE</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/NOTICE</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
<exclude>META-INF/README.md</exclude>
<exclude>META-INF/DEPENDENCIES</exclude>
<!-- Dependency metadata (131+ files, useless at runtime) -->
<exclude>META-INF/maven/**</exclude>
<!-- GraalVM native-image config (9 files, not using native image) -->
<exclude>META-INF/native-image/**</exclude>
<!-- ProGuard/R8 keep rules (5 files, not shrinking at runtime) -->
<exclude>META-INF/proguard/**</exclude>
<!-- Kotlin module metadata (17 files, not needed for reflection) -->
<exclude>META-INF/*.kotlin_module</exclude>
<!-- R8 shrinker bundled inside dex2jar's d2j-external (~13.8MB unpacked) -->
<exclude>com/android/tools/r8/**</exclude>
<!-- ICU4J locale data pulled in by ANTLR4 (~22MB unpacked) -->
<exclude>com/ibm/icu/impl/data/**</exclude>
<!-- ANTLR4 Swing GUI (grammar workbench, ~86KB) -->
<exclude>org/antlr/v4/gui/**</exclude>
<!-- dex2jar/R8 API compatibility database (~1MB) -->
<exclude>api_database/**</exclude>
<!-- ByteBuddy Windows native DLLs (~0.5MB, irrelevant on Linux/Mac) -->
<exclude>win32-x86/**</exclude>
<exclude>win32-x86-64/**</exclude>
</excludes>
</filter>
<filter>
@@ -209,42 +236,34 @@
<exclude>dev/k3vinb5/unyo/core/cli/**</exclude>
</excludes>
</filter>
<!--
The following force-include filters were needed when minimizeJar=true
to prevent maven-shade-plugin from stripping classes loaded via
reflection at runtime. With minimizeJar=false they are redundant
but kept commented for reference.
-->
<!--
<filter>
<artifact>dev.k3vinb5:androidcompat</artifact>
<includes>
<include>**</include>
</includes>
<includes><include>**</include></includes>
</filter>
<!--
Force-include all Kotlin runtime classes.
Extensions are loaded dynamically and reference internal Kotlin
classes (e.g. SpillingKt) that our core code never directly
imports. Without this, maven-shade-plugin strips them.
-->
<filter>
<artifact>org.jetbrains.kotlin:*</artifact>
<includes>
<include>**</include>
</includes>
<includes><include>**</include></includes>
</filter>
<filter>
<artifact>org.jetbrains.kotlinx:*</artifact>
<includes>
<include>**</include>
</includes>
<includes><include>**</include></includes>
</filter>
<!--
Force-include org.json classes.
Extensions dynamically use org.json.JSONObject and the Android SDK
org/json stubs are excluded from android.jar unpack. Without this,
maven-shade-plugin strips the real org.json implementation.
-->
<filter>
<artifact>org.json:json</artifact>
<includes>
<include>**</include>
</includes>
<includes><include>**</include></includes>
</filter>
<filter>
<artifact>commons-logging:commons-logging</artifact>
<includes><include>**</include></includes>
</filter>
-->
</filters>
</configuration>
</execution>

6
aniyomi-bridge/src/main/kotlin/eu/kanade/tachiyomi/network/NetworkHelper.kt
View File

@@ -22,9 +22,9 @@ class NetworkHelper(
private val clientBuilder: OkHttpClient.Builder = run {
val builder = OkHttpClient.Builder()
.cookieJar(cookieJar)
.connectTimeout(120, TimeUnit.SECONDS)
.readTimeout(120, TimeUnit.SECONDS)
.callTimeout(120, TimeUnit.SECONDS)
.connectTimeout(15, TimeUnit.SECONDS)
.readTimeout(30, TimeUnit.SECONDS)
.callTimeout(40, TimeUnit.SECONDS)
.dispatcher(Dispatcher().apply { maxRequestsPerHost = 10 })
.cache(
Cache(

132
aniyomi-bridge/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareCookieCache.kt Normal file
View File

@@ -0,0 +1,132 @@
package eu.kanade.tachiyomi.network.interceptor
import android.app.Application
import android.content.SharedPreferences
import eu.kanade.tachiyomi.network.DesktopCookieJar
import kotlinx.serialization.Serializable
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import okhttp3.Cookie
import okhttp3.HttpUrl.Companion.toHttpUrl
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
/**
* Persistent cache for Cloudflare clearance cookies.
*
* When a Cloudflare challenge is solved via WebView the resulting cookies
* (especially **cf_clearance**) are stored here keyed by domain.
* On subsequent requests the cached cookies are injected into the
* [DesktopCookieJar] before attempting the WebView challenge, which
* allows the request to sail through when the cookie is still valid
* (~24 h on Cloudflare’s side, we conservatively cache for 12 h).
*/
object CloudflareCookieCache {
private const val PREFS_NAME = "cloudflare_cookie_cache"
private const val KEY_PREFIX = "cookies_"
/** Max age in milliseconds — Cloudflare usually honours cf_clearance for ~24 h. */
private const val MAX_CACHE_AGE_MS = 12 * 60 * 60 * 1000L // 12 hours
private val json = Json { ignoreUnknownKeys = true }
private val prefs: SharedPreferences by lazy {
Injekt.get<Application>().getSharedPreferences(PREFS_NAME, android.content.Context.MODE_PRIVATE)
}
/**
* Returns the cached cookies for [domain] if they exist and are not stale.
*/
fun get(domain: String): List<CachedCookie>? {
val raw = prefs.getString(KEY_PREFIX + domain, null) ?: return null
val entry = try {
json.decodeFromString<CacheEntry>(raw)
} catch (_: Exception) {
prefs.edit().remove(KEY_PREFIX + domain).apply()
return null
}
val age = System.currentTimeMillis() - entry.timestamp
if (age > MAX_CACHE_AGE_MS) {
prefs.edit().remove(KEY_PREFIX + domain).apply()
return null
}
return entry.cookies
}
/**
* Stores cookies for [domain] replacing any previous entry.
*/
fun put(domain: String, cookies: List<Cookie>) {
val entry = CacheEntry(
timestamp = System.currentTimeMillis(),
cookies = cookies.map { CachedCookie.fromOkHttp(it) }
)
prefs.edit().putString(KEY_PREFIX + domain, json.encodeToString(entry)).apply()
}
/**
* Injects cached cookies into [cookieJar] for [url].
* Returns **true** if cookies were injected.
*/
fun injectIntoJar(cookieJar: DesktopCookieJar, url: String): Boolean {
val domain = url.toHttpUrl().host
val cached = get(domain) ?: return false
val fakeUrl = "https://$domain/".toHttpUrl()
val cookies = cached.mapNotNull { it.toOkHttp(fakeUrl) }
if (cookies.isEmpty()) return false
cookieJar.saveFromResponse(fakeUrl, cookies)
return true
}
@Serializable
private data class CacheEntry(
val timestamp: Long,
val cookies: List<CachedCookie>,
)
@Serializable
data class CachedCookie(
val name: String,
val value: String,
val domain: String? = null,
val path: String? = null,
val maxAge: Long = -1,
val secure: Boolean = false,
val httpOnly: Boolean = false,
) {
companion object {
fun fromOkHttp(c: Cookie) = CachedCookie(
name = c.name,
value = c.value,
domain = c.domain,
path = c.path,
maxAge = c.expiresAt,
secure = c.secure,
httpOnly = c.httpOnly,
)
}
fun toOkHttp(url: okhttp3.HttpUrl): Cookie? {
val builder = Cookie.Builder()
.name(name)
.value(value)
.path(path ?: "/")
domain?.let { builder.domain(it) }
if (secure) builder.secure()
if (httpOnly) builder.httpOnly()
if (maxAge > 0) builder.expiresAt(maxAge)
return try {
builder.build()
} catch (_: IllegalArgumentException) {
null
}
}
}
}

23
aniyomi-bridge/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/CloudflareInterceptor.kt
View File

@@ -56,8 +56,29 @@ class CloudflareInterceptor(
cookieManager.remove(request.url, COOKIE_NAMES, 0)
val oldCookie = cookieManager.get(request.url)
.firstOrNull { it.name == "cf_clearance" }
// Fast path: try cached clearance cookies before burning time on WebView
val cachedInjected = CloudflareCookieCache.injectIntoJar(cookieManager, request.url.toString())
if (cachedInjected) {
val cachedResponse = chain.proceed(request)
if (!shouldIntercept(cachedResponse)) {
// Cache hit — no challenge, return immediately
return cachedResponse
}
// Cookie expired or rejected, close and fall through to WebView
cachedResponse.close()
cookieManager.remove(request.url, COOKIE_NAMES, 0)
}
resolveWithWebView(request, oldCookie)
// Persist the freshly-obtained cookies so the next request is fast
val freshCookies = cookieManager.get(request.url)
.filter { it.name == "cf_clearance" || it.name.startsWith("cf_") }
if (freshCookies.isNotEmpty()) {
CloudflareCookieCache.put(request.url.host, freshCookies)
}
return chain.proceed(request)
}
// Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
@@ -126,7 +147,7 @@ class CloudflareInterceptor(
webview?.loadUrl(origRequestUrl, headers)
}
latch.awaitFor30Seconds()
latch.awaitFor20Seconds()
executor.execute {
if (!cloudflareBypassed) {

4
aniyomi-bridge/src/main/kotlin/eu/kanade/tachiyomi/network/interceptor/WebViewInterceptor.kt
View File

@@ -78,8 +78,8 @@ abstract class WebViewInterceptor(
.mapValues { it.value.getOrNull(0).orEmpty() }
}
fun CountDownLatch.awaitFor30Seconds() {
await(30, TimeUnit.SECONDS)
fun CountDownLatch.awaitFor20Seconds() {
await(20, TimeUnit.SECONDS)
}
fun createWebView(request: Request): WebView {

2
pom.xml
View File

@@ -27,7 +27,7 @@
<okhttp.version>5.0.0-alpha.14</okhttp.version>
<gson.version>2.11.0</gson.version>
<jsoup.version>1.17.2</jsoup.version>
<htmlunit.version>4.6.0</htmlunit.version>
<htmlunit.version>4.13.0</htmlunit.version>
<rxjava.version>1.1.3</rxjava.version>
<rxrelay.version>1.2.0</rxrelay.version>
<jboss-logging.version>3.6.1.Final</jboss-logging.version>