From df49a1175eb93f2cdae99ea2362f83d2155cfefe Mon Sep 17 00:00:00 2001 From: Karlatemp Date: Mon, 28 Nov 2022 00:09:37 +0800 Subject: [PATCH] fixa --- .github/workflows/release.yml | 110 +++++-------- buildSrc/src/main/kotlin/GpgSigner.kt | 24 --- ci-release-helper/build.gradle.kts | 5 + ci-release-helper/src/CiHelper.kt | 224 +++++++++++++++++++++----- 4 files changed, 231 insertions(+), 132 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ab7cfde25..7f5ac8a9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,32 @@ on: jobs: + initialize-sonatype-stage: + name: "Initialize sonatype stage repository" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: '17' + + - name: Setup Gradle + uses: gradle/gradle-build-action@v2 + + - run: chmod -R 777 * + + - name: Publish to maven central + run: ./gradlew runcihelper --args create-stage-repo --scan "-Pcihelper.cert.username=${{ secrets.SONATYPE_USER }}" "-Pcihelper.cert.password=${{ secrets.SONATYPE_KEY }}" "-Pcihelper.cert.profileid=${{ secrets.SONATYPE_PROFILEID }}" + + - name: Cache stage repo id + uses: actions/upload-artifact@v3 + with: + name: publish-stage-id + path: ci-release-helper/repoid publish-others: name: "Others (${{ matrix.os }})" @@ -32,25 +58,20 @@ jobs: with: submodules: 'recursive' - - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: '17' - - name: Keys setup shell: bash run: | mkdir build-gpg-sign echo "$GPG_PRIVATE" > build-gpg-sign/keys.gpg echo "$GPG_PUBLIC_" > build-gpg-sign/keys.gpg.pub - mkdir build-secret-keys - echo "$SONATYPE_USER" > build-secret-keys/sonatype.key - echo "$SONATYPE_KEY" >> build-secret-keys/sonatype.key env: GPG_PRIVATE: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PUBLIC_: ${{ secrets.GPG_PUBLIC_KEY }} - SONATYPE_USER: ${{ secrets.SONATYPE_USER }} - SONATYPE_KEY: ${{ secrets.SONATYPE_KEY }} + + - uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: '17' - name: Setup Gradle uses: gradle/gradle-build-action@v2 @@ -81,9 +102,6 @@ jobs: - name: Clean and download dependencies run: ./gradlew clean ${{ env.gradleArgs }} - - name: Check keys - run: ./gradlew ensureMavenCentralAvailable ${{ env.gradleArgs }} - - name: "Assemble" run: ./gradlew assemble ${{ env.gradleArgs }} @@ -105,11 +123,14 @@ jobs: if: ${{ env.isMac == 'true' }} run: ./gradlew publishAllPublicationsToMiraiStageRepoRepository ${{ env.gradleArgs }} - - name: Publish publish stage - uses: actions/upload-artifact@v3 + - name: Restore stage repo id + uses: actions/download-artifact@v3 with: - name: publish-stage-1 - path: ci-release-helper/stage-repo + name: publish-stage-id + path: ci-release-helper/repoid + + - name: Publish to maven central + run: ./gradlew runcihelper --args publish-to-maven-central --scan "-Pcihelper.cert.username=${{ secrets.SONATYPE_USER }}" "-Pcihelper.cert.password=${{ secrets.SONATYPE_KEY }}" - name: Publish Gradle plugin run: ./gradlew @@ -164,14 +185,9 @@ jobs: mkdir build-gpg-sign echo "$GPG_PRIVATE" > build-gpg-sign/keys.gpg echo "$GPG_PUBLIC_" > build-gpg-sign/keys.gpg.pub - mkdir build-secret-keys - echo "$SONATYPE_USER" > build-secret-keys/sonatype.key - echo "$SONATYPE_KEY" >> build-secret-keys/sonatype.key env: GPG_PRIVATE: ${{ secrets.GPG_PRIVATE_KEY }} GPG_PUBLIC_: ${{ secrets.GPG_PUBLIC_KEY }} - SONATYPE_USER: ${{ secrets.SONATYPE_USER }} - SONATYPE_KEY: ${{ secrets.SONATYPE_KEY }} - name: Setup Gradle uses: gradle/gradle-build-action@v2 @@ -238,9 +254,6 @@ jobs: - name: Clean and download dependencies run: ./gradlew clean ${{ env.gradleArgs }} - - name: Check keys - run: ./gradlew ensureMavenCentralAvailable ${{ env.gradleArgs }} - - name: "Test mirai-core-utils for ${{ matrix.os }}" run: ./gradlew :mirai-core-utils:${{ matrix.targetName }}Test ${{ env.gradleArgs }} @@ -269,53 +282,14 @@ jobs: if: ${{ env.isMac == 'true' }} run: ./gradlew publishMacosX64PublicationToMiraiStageRepoRepository ${{ env.gradleArgs }} - - name: Publish publish stage - uses: actions/upload-artifact@v3 - with: - name: publish-stage-2 - path: ci-release-helper/stage-repo - - publish-stage-to-maven-central: - name: "Publish stage to maven central" - needs: - - publish-others - - publish-core-native - runs-un: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - - uses: actions/setup-java@v2 - with: - distribution: 'adopt-openj9' - java-version: '17' - - - - name: Setup Gradle - uses: gradle/gradle-build-action@v2 - - - name: Download publish stage 1 + - name: Restore stage repo id uses: actions/download-artifact@v3 with: - name: publish-stage-1 - path: ci-release-helper/stage-1 - - - name: Download publish stage 2 - uses: actions/download-artifact@v3 - with: - name: publish-stage-1 - path: ci-release-helper/stage-2 - - - name: Merge stages - run: > - ./gradlew runcihelper --args merge-repos --scan - -Pcihelper.repo1=ci-release-helper/stage-1 - -Pcihelper.repo2=ci-release-helper/stage-2 + name: publish-stage-id + path: ci-release-helper/repoid - name: Publish to maven central - run: ./gradlew runcihelper --args publish-to-maven-central --scan "-Dcihelper.cert.username=${{ secrets.SONATYPE_USER }}" "-Dcihelper.cert.password=${{ secrets.SONATYPE_KEY }}" + run: ./gradlew runcihelper --args publish-to-maven-central --scan "-Pcihelper.cert.username=${{ secrets.SONATYPE_USER }}" "-Pcihelper.cert.password=${{ secrets.SONATYPE_KEY }}" # # close-repository: diff --git a/buildSrc/src/main/kotlin/GpgSigner.kt b/buildSrc/src/main/kotlin/GpgSigner.kt index 8ee91e941..178c96060 100644 --- a/buildSrc/src/main/kotlin/GpgSigner.kt +++ b/buildSrc/src/main/kotlin/GpgSigner.kt @@ -42,30 +42,6 @@ open class GpgSigner(private val workdir: File) { private var initialized: Boolean = false var signer: GpgSigner = NoopSigner fun setup(project: Project) { - if (initialized) return - initialized = true - val rootProject = project.rootProject - val gpg = rootProject.projectDir.resolve("build-gpg-sign") - gpg.mkdirs() - val keyFile = gpg.resolve("keys.gpg") - val keyFilePub = gpg.resolve("keys.gpg.pub") - if (keyFile.isFile) { - val homedir = gpg.resolve("homedir") - signer = GpgSigner(homedir.absolutePath) - if (!homedir.resolve("pubring.kbx").isFile) { - signer.importKey(keyFile) - if (keyFilePub.isFile) { - signer.importKey(keyFilePub) - } else { - println("[GPG SIGN] Missing public key storage") - println("[GPG SIGN] GPG Sign 2nd verity may failed.") - } - } - } else { - println("[GPG SIGN] GPG Key not found.") - println("[GPG SIGN] GPG Signer will not setup") - println("[GPG SIGN] Key file location: $keyFile") - } } } diff --git a/ci-release-helper/build.gradle.kts b/ci-release-helper/build.gradle.kts index 7dea1f0e7..3cf7c1088 100644 --- a/ci-release-helper/build.gradle.kts +++ b/ci-release-helper/build.gradle.kts @@ -18,6 +18,7 @@ plugins { tasks.register("runcihelper") { this.classpath = sourceSets["main"].runtimeClasspath this.mainClass.set("cihelper.CiHelperKt") + this.workingDir = rootProject.projectDir fun Project.findPublishingExt(): PublishingExtension? { val exts = (this@findPublishingExt as ExtensionAware).extensions @@ -64,6 +65,10 @@ nexusStaging { password = keys.password } +dependencies { + implementation(`kotlinx-serialization-json`) +} + tasks.register("updateSnapshotVersion") { group = "mirai" diff --git a/ci-release-helper/src/CiHelper.kt b/ci-release-helper/src/CiHelper.kt index fe10a067f..402822566 100644 --- a/ci-release-helper/src/CiHelper.kt +++ b/ci-release-helper/src/CiHelper.kt @@ -10,17 +10,51 @@ package cihelper +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import java.io.File +import java.io.OutputStream import java.net.URI import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse import java.nio.charset.Charset +import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Paths +import java.nio.file.attribute.PosixFilePermission +import java.nio.file.attribute.PosixFilePermissions +import java.security.MessageDigest import java.util.* import java.util.stream.Collectors import kotlin.io.path.* +private val hexTemplate: CharArray = "0123456789abcdef".toCharArray() +private const val useragent = "Gradle/7.3.1 (Windows 10;10.0;amd64) (Azul Systems, Inc.;18.0.2.1;18.0.2.1+1)" + +fun ByteArray.hexToString(): String { + val sb = StringBuilder(this.size * 2) + forEach { sbyte -> + sb.append(hexTemplate[sbyte.toInt().shr(4).and(0xF)]) + sb.append(hexTemplate[sbyte.toInt().and(0xF)]) + } + return sb.toString() +} + +private fun getAuth(): String { + + val cert_username = + System.getenv("CERT_USERNAME") ?: System.getProperty("cihelper.cert.username") ?: error("CERT_USERNAME") + val cert_password = + System.getenv("CERT_PASSWORD") ?: System.getProperty("cihelper.cert.password") ?: error("CERT_PASSWORD") + + return "Basic " + Base64.getEncoder().encodeToString( + ("$cert_username:$cert_password").toByteArray() + ) +} + @Suppress("Since15") fun main(args: Array) { val projVer = System.getenv("PROJ_VERSION") ?: error("Please use `./gradlew runcihelper --args XXXX`") @@ -73,41 +107,35 @@ fun main(args: Array) { } } - "merge-repos" -> { - val repos = System.getProperties().asSequence() - .filter { it.key.toString().startsWith("cihelper.repo") } - .map { it.value.toString() } - .map { Paths.get(it) } - .toList() - println(repos) - - repos.forEach { arepo -> - Files.walk(arepo) - .filter { it.fileName.name == "maven-metadata.xml" } - .map { it.parent } - .filter { it.resolve(projVer).isDirectory() } - .filter { Files.createDirectories(it.resolve(projVer)).iterator().hasNext() } - .flatMap { Files.walk(it) } - .filter { it.isRegularFile() } - .forEach { spath -> - val rel = arepo.relativize(spath) - - val target = repoLoc.resolve(rel) - target.parent?.createDirectories() - - println("Copying $spath to $target") - - spath.copyTo(target, overwrite = true) - } + "create-stage-repo" -> { + val rsp = httpc.send( + HttpRequest.newBuilder( + URI.create( + "https://oss.sonatype.org/service/local/staging/profiles/${ + System.getProperty( + "cihelper.cert.profileid" + ) + }/start" + ) + ) + .header("User-Agent", useragent) + .header("Authorization", getAuth()) + .header("Content-Type", "application/json;charset=utf-8") + .POST(HttpRequest.BodyPublishers.ofString("{\"data\":{\"description\": \"mamoe/mirai release $projVer\"}}")) + .build(), + HttpResponse.BodyHandlers.ofString() + ) + if (rsp.statusCode() != 201) { + error(rsp.toString()) } + val rspx = Json.decodeFromString(JsonObject.serializer(), rsp.body()) + val stagedRepositoryId = rspx["data"]!!.jsonObject["stagedRepositoryId"]!!.jsonPrimitive.content + + File("ci-release-helper").also { it.mkdirs() } + .resolve("repoid").writeText(stagedRepositoryId) } "publish-to-maven-central" -> { - val cert_username = - System.getenv("CERT_USERNAME") ?: System.getProperty("cihelper.cert.username") ?: error("CERT_USERNAME") - val cert_password = - System.getenv("CERT_PASSWORD") ?: System.getProperty("cihelper.cert.password") ?: error("CERT_PASSWORD") - // https://oss.sonatype.org/service/local/staging/deploy/maven2 relatedRepoLoc.listDirectoryEntries().forEach { subdir -> val verpath = subdir.resolve(projVer) @@ -120,25 +148,139 @@ fun main(args: Array) { subdir.toFile().deleteRecursively() } } - val pendingFiles = Files.walk(relatedRepoLoc).filter { it.isRegularFile() }.use { stream -> - stream.collect(Collectors.toList()) + val pendingFiles = Files.walk(relatedRepoLoc) + .filter { it.isRegularFile() } + .filter { !it.name.endsWith(".md5") && !it.name.endsWith(".sha1") } + .filter { !it.name.endsWith(".asc") } + .use { stream -> stream.collect(Collectors.toList()) } + + run `sign artifacts`@{ + // build-gpg-sign/keys.gpg + // build-gpg-sign/keys.gpg.pub + val bgs = Paths.get("build-gpg-sign").toAbsolutePath() + if (!bgs.isDirectory()) return@`sign artifacts` + val gpgHomeDir = bgs.resolve("homedir") + val bgsFile = bgs.toFile() + + fun execGpg(vararg cmd: String) { + println("::group::${cmd.joinToString(" ")}") + try { + val exitcode = ProcessBuilder("gpg", "--homedir", "homedir", "--batch", "--no-tty", *cmd) + .directory(bgsFile) + .inheritIO() + .start() + .waitFor() + if (exitcode != 0) { + error("Exit code $exitcode != 0") + } + } finally { + println("::endgroup::") + } + } + + + if (!gpgHomeDir.resolve("pubring.kbx").exists()) { + + val keys = arrayOf("keys.gpg", "keys.gpg.pub") + if (!keys.all { bgs.resolve(it).isRegularFile() }) return@`sign artifacts` + + val isPosix = FileSystems.getDefault().supportedFileAttributeViews().contains("posix") + val dirPermissions = PosixFilePermissions.asFileAttribute( + EnumSet.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE + ) + ) + + Files.createDirectories( + gpgHomeDir, + *if (isPosix) arrayOf(dirPermissions) else arrayOf(), + ) + + keys.forEach { execGpg("--import", it) } + } + + + println("::group::Signing artifacts") + pendingFiles.toList().asSequence().filterNot { it.name == "maven-metadata.xml" } + .forEach { pendingFile -> + val pt = pendingFile.absolutePathString() + val ascFile = pendingFile.resolveSibling(pendingFile.name + ".asc") + ascFile.deleteIfExists() + execGpg("-a", "--detach-sig", "--sign", pt) + + pendingFiles.add(ascFile) + } + println("::endgroup::") + } + + run `calc msg digest`@{ + pendingFiles.toList().forEach { pendingFile -> + val sha1MD = MessageDigest.getInstance("SHA-1") + val md5MD = MessageDigest.getInstance("MD5") + + pendingFile.inputStream().use { content -> + content.copyTo(object : OutputStream() { + override fun write(b: Int) { + sha1MD.update(b.toByte()) + md5MD.update(b.toByte()) + } + + override fun write(b: ByteArray, off: Int, len: Int) { + sha1MD.update(b, off, len) + md5MD.update(b, off, len) + } + }) + } + + val sha1 = sha1MD.digest().hexToString() + val mg5 = md5MD.digest().hexToString() + + val pfname = pendingFile.name + val sha1File = pendingFile.resolveSibling("$pfname.sha1") + val md5File = pendingFile.resolveSibling("$pfname.md5") + + sha1File.writeText(sha1) + md5File.writeText(mg5) + + pendingFiles.add(sha1File) + pendingFiles.add(md5File) + } } - // TODO: Sign flies direct, calc md5, sha1 direct pendingFiles.sort() + println("::group::Publishing to Maven Central") - val authorization = "Basic " + Base64.getEncoder().encodeToString( - ("$cert_username:$cert_password").toByteArray() - ) - val useragent = "Gradle/7.3.1 (Windows 10;10.0;amd64) (Azul Systems, Inc.;18.0.2.1;18.0.2.1+1)" + val authorization = getAuth() val errors = mutableListOf() + fun resolveSonatypeRepoLoc(): String { + val repoIdPath = Paths.get("ci-release-helper/repoid") + + var repoId = "" + if (repoIdPath.isRegularFile()) { + repoId = repoIdPath.readText().trim() + } else if (repoIdPath.isDirectory()) { + val files = repoIdPath.listDirectoryEntries().filter { it.isRegularFile() } + if (files.size == 1) { + repoId = files.first().readText().trim() + } + } + if (repoId.isNotBlank()) { + return "https://oss.sonatype.org/service/local/staging/deployByRepositoryId/$repoId/" + } + + return "https://oss.sonatype.org/service/local/staging/deploy/maven2/" + } + + val repoServerLocation = resolveSonatypeRepoLoc() + pendingFiles.forEach { pending -> val netpath = repoLoc.relativize(pending) - val uri = "https://oss.sonatype.org/service/local/staging/deploy/maven2/" + (netpath.toString() - .replace("\\", "/")) + val uri = repoServerLocation + (netpath.toString().replace("\\", "/")) println("Processing $uri") @@ -157,6 +299,8 @@ fun main(args: Array) { println(errmsg) } } + + println("::endgroup::") if (errors.isNotEmpty()) { error(errors.joinToString("\n\n", prefix = "\n")) }