Support AES-128 HLS cache decryption

Closes #2881
This commit is contained in:
Him188
2026-03-13 04:15:16 +09:00
parent 746db73ffe
commit 86ae533175
6 changed files with 265 additions and 12 deletions

View File

@@ -44,6 +44,7 @@ kotlin {
api(projects.utils.coroutines)
api(libs.kotlinx.datetime)
api(libs.mediamp.ffmpeg)
implementation(libs.korlibs.crypto)
implementation(projects.utils.logging)
implementation(projects.utils.ktorClient)
api(libs.datastore.core)
@@ -64,6 +65,14 @@ kotlin {
}
dependencies {
when (val triple = getOsTriple()) {
"windows-x64" -> add("desktopTestRuntimeOnly", libs.mediamp.ffmpeg.runtime.windows.x64)
"linux-x64" -> add("desktopTestRuntimeOnly", libs.mediamp.ffmpeg.runtime.linux.x64)
"macos-x64" -> add("desktopTestRuntimeOnly", libs.mediamp.ffmpeg.runtime.macos.x64)
"macos-arm64" -> add("desktopTestRuntimeOnly", libs.mediamp.ffmpeg.runtime.macos.arm64)
else -> throw UnsupportedOperationException("Unknown os: $triple")
}
"iosArm64FfmpegRuntime"(libs.mediamp.ffmpeg.runtime.ios.arm64)
"iosSimulatorArm64FfmpegRuntime"(libs.mediamp.ffmpeg.runtime.ios.simulator.arm64)
}

View File

@@ -217,12 +217,23 @@ data class SegmentInfo(
val url: String,
val isDownloaded: Boolean,
val byteSize: Long = -1,
val durationSeconds: Float? = null,
val title: String? = null,
val isDiscontinuity: Boolean = false,
val encryption: SegmentEncryptionInfo? = null,
@SerialName("tempFilePath")
val relativeTempFilePath: String,
val rangeStart: Long? = null,
val rangeEnd: Long? = null,
)
@Serializable
data class SegmentEncryptionInfo(
val method: String,
val keyUri: String,
val iv: String? = null,
)
@Serializable
data class DownloadOptions(
val maxConcurrentSegments: Int = 3,

View File

@@ -9,6 +9,7 @@
package me.him188.ani.utils.httpdownloader
import io.ktor.client.call.body
import io.ktor.client.request.header
import io.ktor.client.request.prepareGet
import io.ktor.client.statement.HttpStatement
@@ -55,6 +56,7 @@ import me.him188.ani.utils.httpdownloader.DownloadStatus.FAILED
import me.him188.ani.utils.httpdownloader.DownloadStatus.INITIALIZING
import me.him188.ani.utils.httpdownloader.DownloadStatus.MERGING
import me.him188.ani.utils.httpdownloader.DownloadStatus.PAUSED
import me.him188.ani.utils.httpdownloader.m3u.MediaSegmentEncryption
import me.him188.ani.utils.httpdownloader.m3u.DefaultM3u8Parser
import me.him188.ani.utils.httpdownloader.m3u.M3u8Parser
import me.him188.ani.utils.httpdownloader.m3u.M3u8Playlist
@@ -75,10 +77,13 @@ import me.him188.ani.utils.logging.logger
import me.him188.ani.utils.logging.trace
import me.him188.ani.utils.logging.warn
import me.him188.ani.utils.platform.Uuid
import korlibs.crypto.AES
import korlibs.crypto.CipherPadding
import org.openani.mediamp.ffmpeg.FFmpegKit
import kotlin.concurrent.atomics.AtomicLong
import kotlin.concurrent.atomics.ExperimentalAtomicApi
import kotlin.coroutines.CoroutineContext
import kotlin.text.hexToByteArray
import kotlin.time.Clock
/**
@@ -722,6 +727,7 @@ open class KtorHttpDownloader(
protected suspend fun downloadSingleSegment(
segmentInfo: SegmentInfo,
options: DownloadOptions,
loadEncryptionKey: suspend (SegmentEncryptionInfo) -> ByteArray = { error("No encryption key loader provided") },
): Long {
// If we have a range, add it to the request headers.
val finalOptions = if (segmentInfo.rangeStart != null && segmentInfo.rangeEnd != null) {
@@ -735,8 +741,6 @@ open class KtorHttpDownloader(
return httpGet(segmentInfo.url, finalOptions) { statement ->
val response = statement.execute()
val channel = response.bodyAsChannel()
val segmentPath = baseSaveDir.resolve(segmentInfo.relativeTempFilePath)
withContext(ioDispatcher) {
fileSystem.createDirectories(
@@ -744,7 +748,20 @@ open class KtorHttpDownloader(
)
}
copyChannelToFile(channel, segmentPath).also {
val byteSize = if (segmentInfo.encryption == null) {
val channel = response.bodyAsChannel()
copyChannelToFile(channel, segmentPath)
} else {
val encryptedBytes = response.body<ByteArray>()
val decryptedBytes = decryptHlsSegment(
encryptedBytes,
segmentInfo,
loadEncryptionKey(segmentInfo.encryption),
)
writeBytesToFile(segmentPath, decryptedBytes)
decryptedBytes.size.toLong()
}
byteSize.also {
logger.info { "Segment index=${segmentInfo.index} downloaded, size=$it" }
}
}
@@ -773,6 +790,14 @@ open class KtorHttpDownloader(
return totalBytes.load()
}
private suspend fun writeBytesToFile(filePath: Path, data: ByteArray) {
withContext(ioDispatcher) {
fileSystem.sink(filePath).buffered().use { sink ->
sink.write(data, startIndex = 0, endIndex = data.size)
}
}
}
@OptIn(DelicateCoroutinesApi::class, ExperimentalAtomicApi::class)
protected suspend fun downloadSegments(downloadId: DownloadId, options: DownloadOptions) {
val snapshot = getState(downloadId) ?: return
@@ -782,6 +807,8 @@ open class KtorHttpDownloader(
}
logger.info { "Downloading ${snapshot.segments.size} segments for $downloadId with concurrency=${options.maxConcurrentSegments}" }
val semaphore = Semaphore(options.maxConcurrentSegments)
val keyCache = mutableMapOf<String, ByteArray>()
val keyCacheMutex = Mutex()
coroutineScope {
snapshot.segments.forEach { seg ->
@@ -794,7 +821,18 @@ open class KtorHttpDownloader(
maxRetries = options.maxRetriesPerSegment,
baseDelayMillis = options.baseRetryDelayMillis,
) {
downloadSingleSegment(seg, options)
downloadSingleSegment(seg, options) { encryption ->
keyCacheMutex.withLock {
keyCache[encryption.keyUri] ?: httpGet(
encryption.keyUri,
options.copy(headers = snapshot.requestHeaders),
) {
it.body<ByteArray>()
}.also { keyBytes ->
keyCache[encryption.keyUri] = keyBytes
}
}
}
}
markSegmentDownloaded(downloadId, seg.index, newSize)
} finally {
@@ -1046,7 +1084,62 @@ private fun M3u8Playlist.MediaPlaylist.toSegments(resolveSegmentPath: (String) -
url = seg.uri,
isDownloaded = false,
byteSize = seg.byteRange?.length ?: -1,
durationSeconds = seg.duration,
title = seg.title,
isDiscontinuity = seg.isDiscontinuity,
encryption = seg.encryption?.toSegmentEncryptionInfo(),
relativeTempFilePath = resolveSegmentPath("$idx.ts"),
)
}
}
private fun MediaSegmentEncryption.toSegmentEncryptionInfo(): SegmentEncryptionInfo =
SegmentEncryptionInfo(
method = method,
keyUri = uri,
iv = iv,
)
private fun decryptHlsSegment(
encryptedBytes: ByteArray,
segmentInfo: SegmentInfo,
keyBytes: ByteArray,
): ByteArray {
require(keyBytes.size == 16) {
"HLS AES-128 key must be 16 bytes, but was ${keyBytes.size} bytes for ${segmentInfo.url}"
}
val encryption = segmentInfo.encryption ?: return encryptedBytes
require(encryption.method.equals("AES-128", ignoreCase = true)) {
"Unsupported HLS encryption method '${encryption.method}' for ${segmentInfo.url}"
}
return AES.decryptAesCbc(
encryptedBytes,
keyBytes,
encryption.iv.parseHlsIvOrDefault(segmentInfo.index.toLong()),
CipherPadding.PKCS7Padding,
)
}
private fun String?.parseHlsIvOrDefault(sequenceNumber: Long): ByteArray {
if (this == null) {
return sequenceNumber.toHlsIvBytes()
}
val normalized = removePrefix("0x").removePrefix("0X")
require(normalized.length == 32) {
"HLS IV must be 16 bytes (32 hex chars), but was '$this'"
}
return normalized.hexToByteArray()
}
private fun Long.toHlsIvBytes(): ByteArray =
ByteArray(16).also { bytes ->
var value = this
for (index in bytes.lastIndex downTo 8) {
bytes[index] = (value and 0xFF).toByte()
value = value ushr 8
}
}

View File

@@ -74,10 +74,18 @@ data class MediaSegment(
val title: String? = null,
val isDiscontinuity: Boolean = false,
val byteRange: ByteRange? = null,
val keys: Map<String, String> = emptyMap(),
val encryption: MediaSegmentEncryption? = null,
val tags: Map<String, String> = emptyMap(),
)
data class MediaSegmentEncryption(
val method: String,
val uri: String,
val iv: String? = null,
val keyFormat: String? = null,
val keyFormatVersions: String? = null,
)
/**
* Represents a variant stream in a master playlist
*/
@@ -123,8 +131,8 @@ object DefaultM3u8Parser : M3u8Parser {
var currentSegmentTitle: String? = null
var currentSegmentDiscontinuity = false
var currentSegmentByteRange: ByteRange? = null
val currentSegmentKeys = mutableMapOf<String, String>()
val currentSegmentTags = mutableMapOf<String, String>()
var currentSegmentEncryption: MediaSegmentEncryption? = null
// For current variant being built
var currentVariantAttributes = mutableMapOf<String, String>()
@@ -170,7 +178,22 @@ object DefaultM3u8Parser : M3u8Parser {
line.startsWith("#EXT-X-KEY:") -> {
val keyAttributes = parseAttributes(line.substringAfter(":").trim())
currentSegmentKeys.putAll(keyAttributes)
val method = keyAttributes["METHOD"]?.trim().orEmpty()
currentSegmentEncryption = when {
method.isEmpty() -> throw M3uFormatException("Invalid EXT-X-KEY tag: missing METHOD")
method.equals("NONE", ignoreCase = true) -> null
else -> {
val keyUri = keyAttributes["URI"]
?: throw M3uFormatException("Invalid EXT-X-KEY tag: missing URI")
MediaSegmentEncryption(
method = method,
uri = UrlHelpers.computeAbsoluteUrl(baseUrl, keyUri),
iv = keyAttributes["IV"],
keyFormat = keyAttributes["KEYFORMAT"],
keyFormatVersions = keyAttributes["KEYFORMATVERSIONS"],
)
}
}
}
line.startsWith("#EXT-X-STREAM-INF:") -> {
@@ -233,7 +256,7 @@ object DefaultM3u8Parser : M3u8Parser {
title = currentSegmentTitle,
isDiscontinuity = currentSegmentDiscontinuity,
byteRange = currentSegmentByteRange,
keys = currentSegmentKeys.toMap(),
encryption = currentSegmentEncryption,
tags = currentSegmentTags.toMap(),
),
)
@@ -243,7 +266,6 @@ object DefaultM3u8Parser : M3u8Parser {
currentSegmentTitle = null
currentSegmentDiscontinuity = false
currentSegmentByteRange = null
currentSegmentKeys.clear()
currentSegmentTags.clear()
}
}

View File

@@ -39,6 +39,7 @@ import kotlinx.io.files.FileSystem
import kotlinx.io.files.Path
import kotlinx.io.files.SystemFileSystem
import kotlinx.io.files.SystemTemporaryDirectory
import kotlinx.io.readByteArray
import me.him188.ani.utils.io.deleteRecursively
import me.him188.ani.utils.io.resolve
import me.him188.ani.utils.ktor.asScopedHttpClient
@@ -49,11 +50,13 @@ import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
import kotlin.io.encoding.Base64
import kotlin.io.encoding.ExperimentalEncodingApi
import kotlin.time.Clock
import kotlin.time.Duration.Companion.seconds
import kotlin.time.Instant
@OptIn(ExperimentalCoroutinesApi::class)
@OptIn(ExperimentalCoroutinesApi::class, ExperimentalEncodingApi::class)
class KtorHttpDownloaderTest {
private lateinit var testScope: TestScope
private lateinit var testScheduler: TestCoroutineScheduler
@@ -123,6 +126,30 @@ class KtorHttpDownloaderTest {
)
}
"https://example.com/encrypted.m3u8" -> {
respond(
content = ENCRYPTED_MEDIA_PLAYLIST,
status = HttpStatusCode.OK,
headers = headersOf(HttpHeaders.ContentType, "application/vnd.apple.mpegurl"),
)
}
"https://example.com/key.bin" -> {
respond(
content = HLS_ENCRYPTION_KEY,
status = HttpStatusCode.OK,
headers = headersOf(HttpHeaders.ContentType, "application/octet-stream"),
)
}
"https://example.com/encrypted-segment0.ts" -> {
respond(
content = Base64.Default.decode(ENCRYPTED_TS_SEGMENT_BASE64),
status = HttpStatusCode.OK,
headers = headersOf(HttpHeaders.ContentType, "video/mp2t"),
)
}
"https://example.com/error.m3u8" -> {
// 404 response
respond("Not found", HttpStatusCode.NotFound)
@@ -354,6 +381,28 @@ class KtorHttpDownloaderTest {
assertEquals(1024 + 2048 + 3072, outputFileSize, "M3U8 final output file size mismatch.")
}
@Test
fun `download - should merge AES-128 encrypted HLS`() = testScope.runTest {
val downloadId = downloader.downloadWithId(
url = "https://example.com/encrypted.m3u8",
downloadId = DownloadId("encrypted-output"),
)?.downloadId
assertNotNull(downloadId)
downloader.joinDownload(downloadId)
val state = downloader.getState(downloadId)
assertNotNull(state)
assertEquals(DownloadStatus.COMPLETED, state.status)
val outputPath = Path("$tempDir/encrypted-output.mp4")
assertTrue(fileSystem.exists(outputPath), "Expected decrypted HLS output to exist")
assertFalse(fileSystem.exists(Path("$tempDir/segments_$downloadId")), "Merged encrypted cache dir should be removed")
val header = fileSystem.read(outputPath) { readByteArray(8) }
assertEquals("ftyp", header.decodeToString(startIndex = 4, endIndex = 8))
}
@Test
fun `downloadWithId - should use provided ID`() = testScope.runTest {
val customId = DownloadId("custom-test-id")
@@ -1051,6 +1100,69 @@ class KtorHttpDownloaderTest {
#EXT-X-ENDLIST
"""
private const val ENCRYPTED_MEDIA_PLAYLIST = """
#EXTM3U
#EXT-X-VERSION:3
#EXT-X-TARGETDURATION:2
#EXT-X-MEDIA-SEQUENCE:0
#EXT-X-KEY:METHOD=AES-128,URI="https://example.com/key.bin",IV=0x0102030405060708090A0B0C0D0E0F10
#EXTINF:1.0,
https://example.com/encrypted-segment0.ts
#EXT-X-ENDLIST
"""
private val HLS_ENCRYPTION_KEY = byteArrayOf(
0x00, 0x11, 0x22, 0x33,
0x44, 0x55, 0x66, 0x77,
0x88.toByte(), 0x99.toByte(), 0xAA.toByte(), 0xBB.toByte(),
0xCC.toByte(), 0xDD.toByte(), 0xEE.toByte(), 0xFF.toByte(),
)
private val ENCRYPTED_TS_SEGMENT_BASE64 = """
iQvfCN9mTaaYT4glaia0fX3HlZa8qwDHVjKfoaZ8MwUHPfMHixowe2ZBn/KOi5tSi0hZRokYzYRuZY/7
AxRL29c8ovCJ3OJDff30McT1+Flzx5+BXpGnyok9sPEius779jl+ZRltpaCahCUVyEwW6COnyvXbNsdh
hsgmJ7AKIM/MM5Pc+p43KHXC6oyBYd4t0jHGty/qTjIWJuWYLAGE6pNwxBkuVMZqba0rMxdrqrjEhRxT
Ue0pkenCiuZw86zpXAiZHJbt85yGHocMIR7ulTxHyQTK48sPTO5D1IVepzIedvsSh3L3v+UgYLnUwh0K
mD4aLGnf7AiWxiBFHd+BS1twW7urpDe/Pk+/DfsBVVpBwNZSE/p2e3AO/p/7vaWq13JrJAlDIt3NTY46
hIhJLYWt6qj6okl0R30sT3WWVzfrCsmiBUbb7XZKagDVzfGAWvlreffUmuCyQVG85vhskcX+eJ5XqpJu
dAaVcAmVzVUvaRls63ZcFaxKoJQMBBxO6SGIsu3cZi/mybCoYTlvb3U1f3D/eXA9dNy8If5IYtcuBtJr
rrhdzWqVzd89oUkWu2y71re6dVqp5TWBTW+LdqPhMUbOtme1hv13YsQSsgtO056wgrXb1fL4dFJ3tos6
ORTib1djZwD7EprJ01qEITbRtGEe8sWt7eZEwgt3ivpiOTEiwPtcLZNf58MjGWZ5h1e8Kz7RELNhO74j
nmu4Umx/YDtQ3895VvPqCc1SXoprI69mbDo0shhWakgPEMun91iKdpPyxE19d/ra29CnVQgJ6v7Syz32
uB3wlZdPwBf4KSUcsvhL+go95W2v5CTUamfQwVcnccqtALDfDdDa9ZRz9sd6z1YHNVO3UMJLqY4suBrm
Juh+wCxiMBrtAB2BGM3VYxMcApyBUz3iAv88A6PUrDmhBZeCXkue+c9JePjGJX1IaMt3+jyj/Ir5Fod8
5MW2dMj0tLQop3NvnX28KJcrC+9d45N8+rvPfQgOnLoLX2n6j6Cd8INRmeRFHAbhc5v/s+hQU4Rlcd9a
dXUC6AcK8CkbkutM+HVJtbLC6Y5FuoPungEA8D6t/RABbnAq1vBPSujmxOzTURNgUluf2PZuRAF3cwrf
X6tO1w1rRK2zGLrrFmD+EN+DT6Q3YaECSASJboKN8s5PacvYKGSvpLIxvnEuwhJZ3MlhEp/TL/jGU6ar
LH1jCiJD0cVO0D7rhQoGuWatQLM0DlO/rHb4okFhmtJHbOtljx/xHHwyLVS3XMA3WeDBiXbNH2DH2hrV
lCoZ9wkZ/6jF2dFpA1UUwWO4zjZwIa8wDNa4jjm/6aIl7VOZ3uXC7+nLRiV4NI0443tklU5pWcqNIFRj
ZTQlcVm+FEO88p8NfWS6J3X+1cbbHeCsxLFuv0Wva+F3Vjb/06qYo+uN7jj1F0LphYwdj4D6yjcMriEB
Ay/sryHbbLqgjztMExOznS20m+E9H4V/bMhrRIkm2PKn6jRPm1HGBDWQXFfoZ3KwP4utgPx7wmj4hZ7r
xn7TKeQTGklh0wx+0GWROeT4mCFQQ14giVyymrBqz8zWi5mVgEnErC7A4EW6CmsTLTXmmdHc37fGx4G5
CQZ+dgaiHXqO8Gg1mKR3ibXdrfr3Osq+Gyi7mfH/e7cghuv2PvWwGDnG5tZaYyrqyLtll2rPf4xiS/QX
EDcpGyPhrhNlHuOguffD0vYoxs26/QJTTO1kllxRjXPXpCdcX0oBffgfBEO2OsUD3tCTJJQFEgo33yQ9
qs+tCfNfudg307rE2Ix0ZkYR3Z38nnYwZ8H0G2f/gISgpfKHI8mJGZmcXslKGruhXZ5dHD65Y7w5D3uc
pAkOQelck+hLUQnEPKOZTBLorOXp0TRN+TxtI5Fp7OGrah7KvA+l4pcmkMVJNb4GbQkJM54i+37nzb4J
+/XV8E+MwgzQOkF4oxeXl3gXjNAWjIqAnjcqTw4S+jBHajDEkIcE0khLUfkEahCQyXefO5QlMBgJq0Qo
mPxqVzcfJ863A1Wkfc+1jZqb8SQDneAqj1df0posl7THIfl7AE270JUCVfKDuMLJVbz7Cgrdy3JfYmpU
NQwREXJ0GlUtsumkQroZY/DpWT+tFl8nV7xf0zL+MtRChHNZxvO0sx0za7YzkvkCRX0HH9UBQSWa1+eO
M2p3RpK8FBYvSJQP6CnLG0aO7m5UoUSnfG5LsXHOwrIaYHRytx2r0jHerPHGRsHMA9Ph0PGW3yvhdKhz
bthBUpUnYIlFoE/La4lyq1O7WA3q27pLX3pX2J/9oLiCxIkJDzqkC3sHFUhcBiiAWAVH+13hYDLUutry
Y0c74ER4I3lXVv9eS+qPPKYyZUXwe6iH7jwKkuzo7NB3bhPnGdxetiEgXp4vFmE+vjJ981mQpanDBlZj
RJhD5J3jp8Oe8qFdtSk+qOyPOcLoF2j+1K3ZHdhAlTDBcpD3kDsNYX+BIHKrNrtEJmogAd2OYfpjbxwt
IxZKd5HuVQmvd78TlRZbxRqluGhwr3aPlza9NlkPMCjl2imNd+4l70DO+RyEwXCJwMyQx2HEkOA4JLvx
fw/cALGA5K+hAoOvxInnJWobn99YyLGbqIREzlKTpI7gyw437pFzPec15L0cf+scTiO8olknpYa5f7/0
uYf4Ru/0Js3pG3GNGJAzgP9ntfI9lIDTl2UXfufnHAjtHrzSf6trd/5n2UBH//SoDwNP56ys860Ab/6N
yF0As0MucsRES02EoZTpLmEho8zeCSdWhaasweyJDsK3qX71eGL1jJebY9d1ikdMMcCR+dml5boB+h8Q
NjBQ6NvqILr5otvyXl+pk9hyqKq2PaWV5x367QbuVerPQqFB8kwonibq6Pha3p8lwIF2BCKjP8JxuvbP
67JMQFKaDkP7ZMhwCJ15W57BJB5T5ticKM6oMMsExtcnQiq88NmryrV1nZUVvqKaxg5BfpLhcG8Hcwvg
7SEbHckRy/1Hyp/W1UDOQr4Qq5eBQm9/lAorlWHQAw537NwR4bFGmir7WbsiOlTMLeNw2WwW5npNO9Rw
vRrhGRSUz+u7k6v4dri75k2V5aWGv//iQMWceXb5wChClZZ5bYix/6f2qz1XNnIYcQzSTyx1HXQcw1RO
J4UEAtKhYRYOJiFKCrKiMDIiNqeqH6YCkpaeQSiW3/QIk5hsTVwAC2i78SINuvewwUeR+sFS44dq8P3+
yJShVbcQEms+Iau4/jHmCNWNLXtvlhgtIShg5LtpcanagpyAEgcrBYK6f7szwcAy
""".trimIndent().replace("\n", "")
// ------------------------------------------------------------
// NEW: references 2 segments:
// - unstable-segment1.ts => fails first time, then success

View File

@@ -185,8 +185,14 @@ class DefaultM3u8ParserTest {
// Check the encryption key was recorded at the segment level or in the segment's keys map
val segment0 = playlist.segments[0]
assertEquals(8.0f, segment0.duration)
// Key info is in segment0.keys or segment0.tags
assertTrue(segment0.keys.isNotEmpty() || segment0.tags.isNotEmpty())
assertEquals("AES-128", segment0.encryption?.method)
assertEquals("https://keyserver.example.com/key", segment0.encryption?.uri)
assertEquals("0x1A2B3C4D5E6F", segment0.encryption?.iv)
val segment1 = playlist.segments[1]
assertEquals("AES-128", segment1.encryption?.method)
assertEquals("https://keyserver.example.com/key", segment1.encryption?.uri)
assertEquals("0x1A2B3C4D5E6F", segment1.encryption?.iv)
}
@Test