From b227f719a27344caa8e2c669ceb558c76f96baec Mon Sep 17 00:00:00 2001 From: Karlatemp Date: Mon, 5 Apr 2021 22:41:26 +0800 Subject: [PATCH] Fix kt compiler errors (#1159) * Fix kt compiler errors * Revert "Fix compiler bug, fix #1136" This reverts commit 39f6c168 * Fix log style Co-authored-by: Him188 --- .github/workflows/build.yml | 8 +- .github/workflows/doc.yml | 10 +- .github/workflows/release.yml | 4 +- .../mirai-core-api-compileKotlinAndroid.txt | 5 + .../mirai-core-api-compileKotlinJvm.txt | 5 + build.gradle.kts | 1 + buildSrc/build.gradle.kts | 1 + buildSrc/src/main/kotlin/Versions.kt | 1 + buildSrc/src/main/kotlin/analyzes/AsmUtil.kt | 61 ++++++++-- .../kotlin/analyzes/NoSuchMethodAnalyzer.kt | 21 +--- .../kotlin/postktcompile/PostKotlinCompile.kt | 106 ++++++++++++++++++ .../commonMain/kotlin/utils/RemoteFileImpl.kt | 35 ------ 12 files changed, 188 insertions(+), 70 deletions(-) create mode 100644 binary-compatibility-validator/kt-compile-edit/mirai-core-api-compileKotlinAndroid.txt create mode 100644 binary-compatibility-validator/kt-compile-edit/mirai-core-api-compileKotlinJvm.txt create mode 100644 buildSrc/src/main/kotlin/postktcompile/PostKotlinCompile.kt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c673bea10..a333ee296 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,10 +22,10 @@ jobs: run: ./gradlew clean --scan - name: Build mirai-core series - run: ./gradlew assemble --scan + run: ./gradlew assemble -Pmirai.pkc.check.enable --scan - name: mirai-core Tests - run: ./gradlew check --scan + run: ./gradlew check -Pmirai.pkc.check.enable --scan build-all: runs-on: ubuntu-latest @@ -48,7 +48,7 @@ jobs: run: ./gradlew clean --scan - name: Build all - run: ./gradlew assemble --scan + run: ./gradlew assemble -Pmirai.pkc.check.enable --scan - name: All Tests - run: ./gradlew check --scan + run: ./gradlew check -Pmirai.pkc.check.enable --scan diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index c1b30efb5..db0a20ed2 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -23,10 +23,10 @@ jobs: - name: chmod -R 777 * run: chmod -R 777 * - - name: Gradle build - run: ./gradlew clean build # if test's failed, don't publish + - name: Gradle build -Pmirai.pkc.check.enable + run: ./gradlew clean build -Pmirai.pkc.check.enable # if test's failed, don't publish - name: Dokka - run: ./gradlew :mirai-core-api:dokkaHtml + run: ./gradlew :mirai-core-api:dokkaHtml -Pmirai.pkc.check.enable - name: GitHub Pages Deploy uses: peaceiris/actions-gh-pages@v3 with: @@ -55,9 +55,9 @@ jobs: - name: chmod -R 777 * run: chmod -R 777 * - name: Gradle build - run: ./gradlew clean build # if test's failed, don't publish + run: ./gradlew clean build -Pmirai.pkc.check.enable # if test's failed, don't publish - name: Dokka - run: ./gradlew :mirai-console:dokkaHtml + run: ./gradlew :mirai-console:dokkaHtml -Pmirai.pkc.check.enable - name: GitHub Pages Deploy uses: peaceiris/actions-gh-pages@v3 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 22da153f9..a142d8a40 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,10 +62,10 @@ jobs: fillBuildConstants --scan - name: Assemble - run: ./gradlew assemble --scan + run: ./gradlew assemble -Pmirai.pkc.check.enable --scan - name: Check - run: ./gradlew check --scan + run: ./gradlew check -Pmirai.pkc.check.enable --scan - name: Gradle :mirai-core-utils:publish run: > diff --git a/binary-compatibility-validator/kt-compile-edit/mirai-core-api-compileKotlinAndroid.txt b/binary-compatibility-validator/kt-compile-edit/mirai-core-api-compileKotlinAndroid.txt new file mode 100644 index 000000000..0deec4f7a --- /dev/null +++ b/binary-compatibility-validator/kt-compile-edit/mirai-core-api-compileKotlinAndroid.txt @@ -0,0 +1,5 @@ +[net/mamoe/mirai/utils/RemoteFile$isDirectory$1] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.isDirectory(Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 +[net/mamoe/mirai/utils/RemoteFile$listFilesCollection$1] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.listFilesCollection(Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 +[net/mamoe/mirai/utils/RemoteFile$upload$3] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.upload(Ljava/io/File;Lnet/mamoe/mirai/utils/RemoteFile$ProgressionCallback;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 +[net/mamoe/mirai/utils/RemoteFile$upload$5] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.upload(Ljava/io/File;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 +[net/mamoe/mirai/utils/RemoteFile$uploadAndSend$2] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.uploadAndSend(Ljava/io/File;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 diff --git a/binary-compatibility-validator/kt-compile-edit/mirai-core-api-compileKotlinJvm.txt b/binary-compatibility-validator/kt-compile-edit/mirai-core-api-compileKotlinJvm.txt new file mode 100644 index 000000000..0deec4f7a --- /dev/null +++ b/binary-compatibility-validator/kt-compile-edit/mirai-core-api-compileKotlinJvm.txt @@ -0,0 +1,5 @@ +[net/mamoe/mirai/utils/RemoteFile$isDirectory$1] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.isDirectory(Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 +[net/mamoe/mirai/utils/RemoteFile$listFilesCollection$1] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.listFilesCollection(Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 +[net/mamoe/mirai/utils/RemoteFile$upload$3] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.upload(Ljava/io/File;Lnet/mamoe/mirai/utils/RemoteFile$ProgressionCallback;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 +[net/mamoe/mirai/utils/RemoteFile$upload$5] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.upload(Ljava/io/File;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 +[net/mamoe/mirai/utils/RemoteFile$uploadAndSend$2] [invokeSuspend(Ljava/lang/Object;)Ljava/lang/Object;] - Change opcode of net/mamoe/mirai/utils/RemoteFile.uploadAndSend(Ljava/io/File;Lkotlin/coroutines/Continuation;)Ljava/lang/Object; from 183 to 185 diff --git a/build.gradle.kts b/build.gradle.kts index 5563c6e67..115982d72 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -70,6 +70,7 @@ tasks.register("publishMiraiCoreArtifactsToMavenLocal") { } analyzes.CompiledCodeVerify.run { registerAllVerifyTasks() } +postktcompile.PostKotlinCompile.run { registerForAll(rootProject) } allprojects { group = "net.mamoe" diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index ad3797a0f..a61e5a3e5 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -59,4 +59,5 @@ dependencies { api(asm("commons")) api(gradleApi()) + api("com.googlecode.java-diff-utils:diffutils:" + version("difflib")) } \ No newline at end of file diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index f96d39e8d..6c4cd349e 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -43,6 +43,7 @@ object Versions { const val slf4j = "1.7.30" const val log4j = "2.13.3" const val asm = "9.1" + const val difflib = "1.3.0" // If you the versions below, you need to sync changes to mirai-console/buildSrc/src/main/kotlin/Versions.kt diff --git a/buildSrc/src/main/kotlin/analyzes/AsmUtil.kt b/buildSrc/src/main/kotlin/analyzes/AsmUtil.kt index 80e7883cb..628d1dea6 100644 --- a/buildSrc/src/main/kotlin/analyzes/AsmUtil.kt +++ b/buildSrc/src/main/kotlin/analyzes/AsmUtil.kt @@ -14,13 +14,60 @@ import org.objectweb.asm.Opcodes import org.objectweb.asm.tree.ClassNode import org.objectweb.asm.tree.FieldNode import org.objectweb.asm.tree.MethodNode +import java.io.Closeable import java.io.File import java.io.InputStream +import java.util.zip.ZipFile -typealias AsmClasses = Map -typealias AsmClassesM = MutableMap +@Suppress("DELEGATED_MEMBER_HIDES_SUPERTYPE_OVERRIDE") +class AsmClassesM( + var res: List?, + val map: MutableMap> +) : AbstractMap>(), + MutableMap> by map, + Closeable { + operator fun plusAssign(map: AsmClassesM) { + this.map.putAll(map.map) + val r = res + if (r is MutableList) { + r.addAll(map.res ?: emptyList()) + return + } + res = (res ?: emptyList()) + (map.res ?: emptyList()) + } + + override fun close() { + res?.forEach { it.close() } + res = null + map.clear() + } +} + +typealias AsmClasses = AsmClassesM object AsmUtil { + var cc = 0 + fun File.readLib(): AsmClassesM { + val rs = mutableListOf() + val result: AsmClassesM = AsmClassesM(rs, HashMap()) + if (this.name.endsWith(".jar")) { + val zip = ZipFile(this) + rs.add(zip) + zip.entries().iterator().forEach l@{ entry -> + if (entry.isDirectory) return@l + if (!entry.name.endsWith(".class")) return@l + result[entry.name.removePrefix("/").removeSuffix(".class")] = lazy { + zip.getInputStream(entry).use { it.readClass() } + } + } + } else if (this.isDirectory) { + this.walk().filter { it.isFile && it.extension == "class" }.forEach { f -> + f.readClass().let { result[it.name] = lazyOf(it) } + } + } + return result + } + fun ClassNode.getMethod(name: String, desc: String, isStatic: Boolean): MethodNode? { return methods?.firstOrNull { it.name == name && it.desc == desc && ((it.access and Opcodes.ACC_STATIC) != 0) == isStatic @@ -33,7 +80,7 @@ object AsmUtil { } } - fun File.readClass(): ClassNode = inputStream().use { it.readClass() } + fun File.readClass(): ClassNode = inputStream().buffered().use { it.readClass() } fun InputStream.readClass(): ClassNode { val cnode = ClassNode() @@ -46,7 +93,7 @@ object AsmUtil { if (!this.containsKey(owner)) { ClassLoader.getSystemClassLoader().getResourceAsStream("$owner.class")?.use { val c = it.readClass() - this[c.name] = c + this[c.name] = lazyOf(c) } } } @@ -59,7 +106,7 @@ object AsmUtil { opcode: Int ): Boolean { patchJvmClass(owner) - val c = this[owner] ?: return false + val c = this[owner]?.value ?: return false val isStatic = opcode == Opcodes.GETSTATIC || opcode == Opcodes.PUTSTATIC if (c.getField(name, desc, isStatic) != null) { return true @@ -77,7 +124,7 @@ object AsmUtil { patchJvmClass(owner) when (opcode) { Opcodes.INVOKESTATIC -> { - val c = this[owner] ?: return false + val c = this[owner]?.value ?: return false return c.getMethod(name, desc, true) != null } Opcodes.INVOKEINTERFACE, @@ -85,7 +132,7 @@ object AsmUtil { Opcodes.INVOKEVIRTUAL -> { fun loopFind(current: String): Boolean { patchJvmClass(current) - val c = this[current] ?: return false + val c = this[current]?.value ?: return false if (c.getMethod(name, desc, false) != null) return true c.superName?.let { if (loopFind(it)) { diff --git a/buildSrc/src/main/kotlin/analyzes/NoSuchMethodAnalyzer.kt b/buildSrc/src/main/kotlin/analyzes/NoSuchMethodAnalyzer.kt index e7f5db256..68c696830 100644 --- a/buildSrc/src/main/kotlin/analyzes/NoSuchMethodAnalyzer.kt +++ b/buildSrc/src/main/kotlin/analyzes/NoSuchMethodAnalyzer.kt @@ -54,35 +54,22 @@ object NoSuchMethodAnalyzer { fun check(classes: Sequence, libs: Sequence) = AsmUtil.run { val analyzer = AndroidApiLevelCheck.Analyzer(emptyMap()) - val asmClasses: AsmClassesM = mutableMapOf() + val asmClasses: AsmClassesM = AsmClassesM(mutableListOf(), HashMap()) libs.forEach { lib -> - if (lib.name.endsWith(".jar")) { - ZipFile(lib).use { zip -> - zip.entries().iterator().forEach l@{ entry -> - if (entry.isDirectory) return@l - if (!entry.name.endsWith(".class")) return@l - zip.getInputStream(entry).use { it.readClass() }.let { - asmClasses[it.name] = it - } - } - } - } else if (lib.isDirectory) { - lib.walk().filter { it.isFile && it.extension == "class" }.forEach { f -> - f.readClass().let { asmClasses[it.name] = it } - } - } + asmClasses += lib.readLib() } classes.map { it.walk() }.flatten().filter { it.isFile } .filter { it.extension == "class" } .map { it.readClass() to it } .onEach { (c, _) -> - asmClasses[c.name] = c + asmClasses[c.name] = lazyOf(c) }.toList().forEach { (classNode, file) -> analyzer.file = file classNode.methods?.forEach { method -> analyzeMethod(analyzer, method, asmClasses) } } + asmClasses.close() if (analyzer.reported) { error("Verify failed") } diff --git a/buildSrc/src/main/kotlin/postktcompile/PostKotlinCompile.kt b/buildSrc/src/main/kotlin/postktcompile/PostKotlinCompile.kt new file mode 100644 index 000000000..ec5e9d71f --- /dev/null +++ b/buildSrc/src/main/kotlin/postktcompile/PostKotlinCompile.kt @@ -0,0 +1,106 @@ +/* + * Copyright 2019-2021 Mamoe Technologies and contributors. + * + * 此源代码的使用受 GNU AFFERO GENERAL PUBLIC LICENSE version 3 许可证的约束, 可以在以下链接找到该许可证. + * Use of this source code is governed by the GNU AGPLv3 license that can be found through the following link. + * + * https://github.com/mamoe/mirai/blob/master/LICENSE + */ + +package postktcompile + +import analyzes.* +import difflib.DiffUtils +import org.gradle.api.Project +import org.jetbrains.kotlin.gradle.tasks.AbstractKotlinCompileTool +import org.objectweb.asm.ClassWriter +import org.objectweb.asm.Opcodes +import org.objectweb.asm.tree.ClassNode +import org.objectweb.asm.tree.MethodInsnNode +import java.io.File + +object PostKotlinCompile { + private fun AsmClasses.canAccessInterface(from: ClassNode, to: String): Boolean { + fun scan(current: ClassNode): Boolean { + if (current.name == to) return true + if (current.superName == to) return true + current.interfaces?.forEach { itf -> + if (scan(get(itf)?.value ?: return@forEach)) return true + } + return false + } + return scan(from) + } + + fun fixKtClass(dir: File, libs: Set, logs: MutableList) = AsmUtil.run { + val asmClasses: AsmClassesM = AsmClassesM(mutableListOf(), HashMap()) + dir.walk().filter { + it.isFile && it.extension == "class" + }.forEach { f -> + f.readClass().let { asmClasses[it.name] = lazyOf(it) } + } + val classes = asmClasses.values.toList() + libs.forEach { asmClasses += it.readLib() } + + classes.forEach { klassLazy -> + val klass = klassLazy.value + var edited = false + klass.methods?.forEach { method -> + method.instructions?.forEach { insn -> + if (insn is MethodInsnNode) { + if (insn.itf && insn.opcode != Opcodes.INVOKEINTERFACE && insn.opcode != Opcodes.INVOKESTATIC) { + if (!asmClasses.canAccessInterface(klass, insn.owner)) { + // println("${klass.name} . ${method.name}${method.desc}, ${insn.owner}.${insn.name}${insn.desc} (${insn.opcode})") + edited = true + logs.add( + "[${klass.name}] [${method.name}${method.desc}] - Change opcode of ${insn.owner}.${insn.name}${insn.desc} from ${insn.opcode} to ${Opcodes.INVOKEINTERFACE}" + ) + insn.opcode = Opcodes.INVOKEINTERFACE + } + } + } + } + } + if (edited) { + dir.resolve("${klass.name}.class").writeBytes( + ClassWriter(0).also { klass.accept(it) }.toByteArray() + ) + } + } + asmClasses.close() + } + + fun registerForAll(rootProject: Project) { + val validator = rootProject.file("binary-compatibility-validator/kt-compile-edit") + rootProject.subprojects { + val subp: Project = this@subprojects + subp.tasks.withType(AbstractKotlinCompileTool::class.java) { + val task: AbstractKotlinCompileTool<*> = this@withType + task.doLast { + val logFile = validator.resolve("${subp.name.replace(":", "_")}-${task.name}.txt") + val logs = mutableListOf() + println("$task: Pre classes patching......") + task.outputs.files.toList().forEach { f -> + println("$task: Patching $f") + fixKtClass(f, task.classpath.files, logs) + } + println("$task: Kotlin compiled classes fix completed.") + val oldLog = if (logFile.isFile) logFile.readText().trim().lines() else emptyList() + val newLog = logs.sorted() + if (newLog == oldLog) return@doLast + val patch = DiffUtils.diff(oldLog, newLog) + + val diff = DiffUtils.generateUnifiedDiff(logFile.name, logFile.name + ".rebuild", oldLog, patch, 3) + logFile.parentFile.mkdirs() + logFile.writeText(newLog.joinToString("\n", postfix = "\n")) + val diffMsg = diff.joinToString("\n") + if (rootProject.hasProperty("mirai.pkc.check.enable")) { + error(diffMsg) + } else { + println(diffMsg) + } + } + } + } + } +} \ No newline at end of file diff --git a/mirai-core/src/commonMain/kotlin/utils/RemoteFileImpl.kt b/mirai-core/src/commonMain/kotlin/utils/RemoteFileImpl.kt index 3cd845320..eb4e7f0d4 100644 --- a/mirai-core/src/commonMain/kotlin/utils/RemoteFileImpl.kt +++ b/mirai-core/src/commonMain/kotlin/utils/RemoteFileImpl.kt @@ -33,9 +33,7 @@ import net.mamoe.mirai.message.MessageReceipt import net.mamoe.mirai.message.data.FileMessage import net.mamoe.mirai.message.data.sendTo import net.mamoe.mirai.utils.* -import net.mamoe.mirai.utils.ExternalResource.Companion.toExternalResource import net.mamoe.mirai.utils.RemoteFile.Companion.ROOT_PATH -import java.io.File import java.util.* import kotlin.contracts.contract @@ -194,9 +192,6 @@ internal class RemoteFileImpl( } override suspend fun isFile(): Boolean = this.getFileFolderInfo().checkExists(this.path).isFile - - // compiler bug - override suspend fun isDirectory(): Boolean = !isFile() override suspend fun length(): Long = this.getFileFolderInfo().checkExists(this.path).size override suspend fun exists(): Boolean = this.getFileFolderInfo() != null override suspend fun getInfo(): RemoteFile.FileInfo? { @@ -259,9 +254,6 @@ internal class RemoteFileImpl( } } - // compiler bug - override suspend fun listFilesCollection(): List = listFiles().toList() - @Suppress("INVISIBLE_MEMBER", "INVISIBLE_REFERENCE") @OptIn(JavaFriendlyAPI::class) override suspend fun listFilesIterator(lazy: Boolean): Iterator { @@ -313,9 +305,6 @@ internal class RemoteFileImpl( return getFilesFlow().filter { it.id == id }.firstOrNull()?.resolveToFile() } - // compiler bug - override suspend fun resolveById(id: String): RemoteFile? = resolveById(id, deep = true) - override fun resolveSibling(relative: String): RemoteFileImpl { val parent = this.parent if (parent == null) { @@ -524,34 +513,10 @@ internal class RemoteFileImpl( ) } - // compiler bug - override suspend fun upload(resource: ExternalResource): FileMessage { - return upload(resource, null) - } - - // compiler bug - override suspend fun upload(file: File, callback: RemoteFile.ProgressionCallback?): FileMessage = - file.toExternalResource().use { upload(it, callback) } - - //compiler bug - override suspend fun upload(file: File): FileMessage { - // Dear compiler: - // - // Please generate invokeinterface. - // - // Yours Sincerely - // Him188 - return file.toExternalResource().use { upload(it) } - } - override suspend fun uploadAndSend(resource: ExternalResource): MessageReceipt { return upload(resource).sendTo(contact) } - // compiler bug - override suspend fun uploadAndSend(file: File): MessageReceipt = - file.toExternalResource().use { uploadAndSend(it) } - // override suspend fun writeSession(resource: ExternalResource): FileUploadSession { // }