release: 1.8.0-alpha

use mtime and atime instead of ctime
add REC_TTL env variable

.flake8

@@ -1,5 +1,6 @@
 [flake8]
-ignore = D203, W504
+max-line-length = 88
+ignore = D203, W504, W503, E203
 exclude =
     __*,
     .*,

.github/workflows/portable.yml

@@ -40,11 +40,6 @@ jobs:
       - name: Unzip Python archive
         run: Expand-Archive -LiteralPath "python.zip" -DestinationPath "build\python"
 
-      - name: Enter build directory
-        run: |
-          Set-Location -Path "build"
-          ls
-
       - name: Rename ffmpeg directory
         working-directory: build
         run: Rename-Item -Path $($env:FFMPEG_ARCHIVE_NAME).Substring(0, $($env:FFMPEG_ARCHIVE_NAME).Length - 4) "ffmpeg"
@@ -83,16 +78,14 @@ jobs:
         working-directory: build
         run: Copy-Item "${{ github.workspace }}\run.bat" -Destination ".\run.bat"
 
-      - name: Exit build directory
+      - name: Copy run.ps1
         working-directory: build
-        run: |
-          ls
-          Set-Location -Path ".."
+        run: Copy-Item "${{ github.workspace }}\run.ps1" -Destination ".\run.ps1"
 
       - name: Zip files
         run: |
           ls build
-          Compress-Archive -Path @("build\run.bat", "build\python", "build\ffmpeg") -DestinationPath "dist\blrec-${{ github.ref_name }}-win64.zip"
+          Compress-Archive -Path @("build\run.bat", "build\run.ps1", "build\python", "build\ffmpeg") -DestinationPath "dist\blrec-${{ github.ref_name }}-win64.zip"
           ls dist
 
       - name: Upload distributions to release

CHANGELOG.md

@@ -1,5 +1,45 @@
 # 更新日志
 
+## 1.8.0-alpha
+
+- 重构直播流录制
+- 修复一些 bug
+- 优先使用 web api
+
+## 1.7.1
+
+修正 pushkey 验证
+
+## 1.7.0
+
+- 添加封面保存策略
+- 添加 Telegram bot 通知
+- 添加 PushDeer 通知
+- 废弃录制 HLS(ts) 流
+- 在设定时间内没有 fmp4 流自动切换录制 flv 流
+
+### P.S.
+
+录制 fmp4 流基本没什么问题了
+
+录制 fmp4 流基本不受网络波动影响，大概是不会录制到二压画质的。
+
+人气比较高会被二压的直播间大都是有 fmp4 流的。
+
+WEB 端直播播放器是 `Hls7Player` 的直播间支持录制 fmp4 流, `fMp4Player` 则不支持。
+
+## 1.6.2
+
+- 忽略 Windows 注册表 JavaScript MIME 设置 (issue #12, 27)
+- 修复 HLS 录制出错 (issue #39, 41)
+- 修 bug (issue #47)
+- Windows 绿色版默认主机绑定 0.0.0.0 并加上 api key
+
+## 1.6.1
+
+- 修复 bug (issue #37, 38, 40)
+- 接收到错误的数据自动换线路 (issue #43)
+
 ## 1.6.0
 
 - 更新 Pushplus 消息推送 url (issue #26)

FAQ.md

@@ -61,11 +61,11 @@ cpu 使用率过高、网络带宽不足或不稳定、硬盘读写慢都会导
 
 ## 怎样才算是旧录播文件？
 
-创建时间超过 24 小时才会被当成旧录播文件在空间不足时被删除。
+修改时间和访问时间都超过 24 小时才会被当成旧录播文件在空间不足时被删除。
 
 ## 空间不足时是怎样删除旧录播文件的？
 
-删除文件是按创建时间的先后进行的，最早创建的最先被删除，直到可用空间不少于所设置的阈值为止。
+删除文件是按修改时间和访问时间的先后进行的，修改时间或访问时间较早的先被删除，直到可用空间不少于所设置的阈值为止。
 
 ## 支持录制付费直播吗？
 

README.md

@@ -129,7 +129,7 @@ sudo docker run \
 
 例如：`blrec --host 0.0.0.0 --port 8000`
 
-### 安全保障
+### 网络安全
 
 指定 `SSL` 证书使用 **https** 协议并指定 `api key` 可防止被恶意访问和泄漏设置里的敏感信息
 
@@ -141,6 +141,16 @@ sudo docker run \
 
 如果在不信任的环境下，请使用浏览器的隐式模式访问。
 
+### 关于 api-key
+
+api key 可以使用数字和字母，长度限制为最短 8 最长 80。
+
+3 次尝试内 api key 正确客户端 ip 会自动加入白名单，3 次错误后则 ip 会被加入黑名单，黑名单后请求会被拒绝 (403)。
+
+黑名单和白名单数以及同时尝试连接的 ip 数量限制各为 100，黑名单或白名单到达限制后不再接受除了白名单内的其它 ip 。
+
+只有重启才会清空黑名单和白名单。
+
 ## 作为 ASGI 应用运行
 
     uvicorn blrec.web:app

pyproject.toml

@@ -3,4 +3,14 @@ requires = [
     "setuptools >= 57.0.0, < 58.0.0",
     "wheel >= 0.37, < 0.38.0",
 ]
-build-backend = "setuptools.build_meta"
+build-backend = "setuptools.build_meta"
+
+[tool.black]
+line-length = 88
+target-version = ['py38']
+include = '\.py$'
+skip-string-normalization = true
+skip-magic-trailing-comma = true
+
+[tool.isort]
+profile = 'black'

run.bat

@@ -3,20 +3,33 @@ chcp 65001
 
 set PATH=.\ffmpeg\bin;.\python;%PATH%
 
-REM 不使用代理
+@REM 不使用代理
 set no_proxy=*
 
-REM 默认本地主机和端口绑定
-set host=localhost
+@REM 主机和端口绑定，可以按需修改。
+set host=0.0.0.0
 set port=2233
 
-REM 服务器主机和端口绑定，去掉注释并按照自己的情况修改。
-REM set host=0.0.0.0
-REM set port=80
+@REM 关于 api key
+
+@REM api key 可以使用数字和字母，长度限制为最短 8 最长 80。
+
+@REM 3 次尝试内 api key 正确客户端 ip 会自动加入白名单，3 次错误后则 ip 会被加入黑名单，黑名单后请求会被拒绝 (403)。
+
+@REM 黑名单和白名单数以及同时尝试连接的 ip 数量限制各为 100，黑名单或白名单到达限制后不再接受除了白名单内的其它 ip 。
+
+@REM 只有重启才会清空黑名单和白名单。
+
+@REM 浏览器第一次访问会弹对话框要求输入 api key。
+
+@REM 输入的 api key 会被保存在浏览器的 local storage，下次使用同一浏览器不用再次输入。
+
+@REM 请自行修改 api key，不要使用默认的 api key。
+set api_key=bili2233
 
 set DEFAULT_LOG_DIR=日志文件
 set DEFAULT_OUT_DIR=录播文件
 
-python -m blrec -c settings.toml --open --host %host% --port %port%
+python -m blrec -c settings.toml --open --host %host% --port %port% --api-key %api_key%
 
 pause

run.ps1

@@ -0,0 +1,27 @@
+chcp 65001
+
+$env:PATH = ".\ffmpeg\bin;.\python;" + $env:PATH
+
+# 不使用代理
+$env:no_proxy = "*"
+
+# 主机和端口绑定，可以按需修改。
+$env:host = "0.0.0.0"
+$env:port = 2233
+
+# 关于 api key
+# api key 可以使用数字和字母，长度限制为最短 8 最长 80。
+# 3 次尝试内 api key 正确客户端 ip 会自动加入白名单，3 次错误后则 ip 会被加入黑名单，黑名单后请求会被拒绝 (403)。
+# 黑名单和白名单数以及同时尝试连接的 ip 数量限制各为 100，黑名单或白名单到达限制后不再接受除了白名单内的其它 ip 。
+# 只有重启才会清空黑名单和白名单。
+# 浏览器第一次访问会弹对话框要求输入 api key。
+# 输入的 api key 会被保存在浏览器的 local storage，下次使用同一浏览器不用再次输入。
+# 请自行修改 api key，不要使用默认的 api key。
+$env:api_key = "bili2233"
+
+$env:DEFAULT_LOG_DIR = "日志文件"
+$env:DEFAULT_OUT_DIR = "录播文件"
+
+python -m blrec -c settings.toml --open --host $env:host --port $env:port --api-key $env:api_key
+
+pause

setup.cfg

@@ -36,10 +36,11 @@ include_package_data = True
 python_requires = >= 3.8
 install_requires =
     typing-extensions >= 3.10.0.0
+    ordered-set >= 4.1.0, < 5.0.0
     fastapi >= 0.70.0, < 0.71.0
     email_validator >= 1.1.3, < 2.0.0
     click < 8.1.0
-    typer >= 0.4.0, < 0.5.0
+    typer >= 0.4.1, < 0.5.0
     aiohttp >= 3.8.1, < 4.0.0
     requests >= 2.24.0, < 3.0.0
     aiofiles >= 0.8.0, < 0.9.0
@@ -53,7 +54,7 @@ install_requires =
     m3u8 >= 1.0.0, < 2.0.0
     jsonpath == 0.82
     psutil >= 5.8.0, < 6.0.0
-    rx >= 3.2.0, < 4.0.0
+    reactivex >= 4.0.0, < 5.0.0
     bitarray >= 2.2.5, < 3.0.0
     brotli >= 1.0.9, < 2.0.0
     uvicorn[standard] >= 0.15.0, < 0.16.0
@@ -61,7 +62,10 @@ install_requires =
 [options.extras_require]
     dev =
         flake8 >= 4.0.1
-        mypy >= 0.910
+        mypy == 0.910  # https://github.com/samuelcolvin/pydantic/issues/3528
+        isort >= 5.10.1
+        black >= 22.3.0
+        autoflake >= 1.4
 
         setuptools >= 59.4.0
         wheel >= 0.37

src/blrec/__init__.py

@@ -1,4 +1,3 @@
-
 __prog__ = 'blrec'
-__version__ = '1.6.0'
+__version__ = '1.8.0-alpha'
 __github__ = 'https://github.com/acgnhiki/blrec'

src/blrec/application.py

@@ -7,8 +7,7 @@ import attr
 import psutil
 
 from . import __prog__, __version__
-from .flv.data_analyser import MetaData
-from .core.stream_analyzer import StreamProfile
+from .flv.operators import MetaData, StreamProfile
 from .disk_space import SpaceMonitor, SpaceReclaimer
 from .bili.helpers import ensure_room_id
 from .task import (
@@ -31,7 +30,9 @@ from .setting import (
 from .notification import (
     EmailNotifier,
     ServerchanNotifier,
+    PushdeerNotifier,
     PushplusNotifier,
+    TelegramNotifier,
 )
 from .webhook import WebHookEmitter
 
@@ -326,10 +327,14 @@ class Application:
     def _setup_notifiers(self) -> None:
         self._email_notifier = EmailNotifier()
         self._serverchan_notifier = ServerchanNotifier()
+        self._pushdeer_notifier = PushdeerNotifier()
         self._pushplus_notifier = PushplusNotifier()
+        self._telegram_notifier = TelegramNotifier()
         self._settings_manager.apply_email_notification_settings()
         self._settings_manager.apply_serverchan_notification_settings()
+        self._settings_manager.apply_pushdeer_notification_settings()
         self._settings_manager.apply_pushplus_notification_settings()
+        self._settings_manager.apply_telegram_notification_settings()
 
     def _setup_webhooks(self) -> None:
         self._webhook_emitter = WebHookEmitter()
@@ -358,10 +363,14 @@ class Application:
     def _destroy_notifiers(self) -> None:
         self._email_notifier.disable()
         self._serverchan_notifier.disable()
+        self._pushdeer_notifier.disable()
         self._pushplus_notifier.disable()
+        self._telegram_notifier.disable()
         del self._email_notifier
         del self._serverchan_notifier
+        del self._pushdeer_notifier
         del self._pushplus_notifier
+        del self._telegram_notifier
 
     def _destroy_webhooks(self) -> None:
         self._webhook_emitter.disable()

src/blrec/bili/danmaku_client.py

@@ -1,27 +1,22 @@
-import json
-import struct
 import asyncio
+import json
 import logging
-from enum import IntEnum, Enum
+import struct
 from contextlib import suppress
-from typing import Any, Dict, Final, Tuple, List, Union, cast, Optional
+from enum import Enum, IntEnum
+from typing import Any, Dict, Final, List, Optional, Tuple, Union, cast
 
 import aiohttp
-from aiohttp import ClientSession
 import brotli
-from tenacity import (
-    retry,
-    wait_exponential,
-    retry_if_exception_type,
-)
+from aiohttp import ClientSession
+from tenacity import retry, retry_if_exception_type, wait_exponential
 
-from .api import AppApi, WebApi
-from .typing import Danmaku
-from ..event.event_emitter import EventListener, EventEmitter
+from ..event.event_emitter import EventEmitter, EventListener
 from ..exception import exception_callback
-from ..utils.mixins import AsyncStoppableMixin
 from ..logging.room_id import aio_task_with_room_id
-
+from ..utils.mixins import AsyncStoppableMixin
+from .api import AppApi, WebApi
+from .typing import ApiPlatform, Danmaku
 
 __all__ = 'DanmakuClient', 'DanmakuListener', 'Danmaku', 'DanmakuCommand'
 
@@ -64,6 +59,7 @@ class DanmakuClient(EventEmitter[DanmakuListener], AsyncStoppableMixin):
         self.webapi = webapi
         self._room_id = room_id
 
+        self._api_platform: ApiPlatform = 'web'
         self._host_index: int = 0
         self._retry_delay: int = 0
         self._MAX_RETRIES: Final[int] = max_retries
@@ -90,51 +86,72 @@ class DanmakuClient(EventEmitter[DanmakuListener], AsyncStoppableMixin):
 
     @retry(
         wait=wait_exponential(multiplier=0.1, max=60),
-        retry=retry_if_exception_type((
-            asyncio.TimeoutError, aiohttp.ClientError,
-        )),
+        retry=retry_if_exception_type((asyncio.TimeoutError, aiohttp.ClientError)),
     )
     async def _connect(self) -> None:
         logger.debug('Connecting to server...')
-        await self._connect_websocket()
-        await self._send_auth()
-        reply = await self._recieve_auth_reply()
-        await self._handle_auth_reply(reply)
-        logger.debug('Connected to server')
-        await self._emit('client_connected')
+        try:
+            await self._connect_websocket()
+            await self._send_auth()
+            reply = await self._recieve_auth_reply()
+            await self._handle_auth_reply(reply)
+        except Exception:
+            self._rotate_api_platform()
+            await self._update_danmu_info()
+            raise
+        else:
+            logger.debug('Connected to server')
+            await self._emit('client_connected')
 
     async def _connect_websocket(self) -> None:
         url = 'wss://{}:{}/sub'.format(
             self._danmu_info['host_list'][self._host_index]['host'],
             self._danmu_info['host_list'][self._host_index]['wss_port'],
         )
+        logger.debug(f'Connecting WebSocket... {url}')
         try:
             self._ws = await self.session.ws_connect(url, timeout=5)
-        except BaseException:
+        except Exception as exc:
+            logger.debug(f'Failed to connect WebSocket: {repr(exc)}')
             host_count = len(self._danmu_info['host_list'])
             self._host_index = (self._host_index + 1) % host_count
             raise
-        logger.debug('Established WebSocket connection')
+        else:
+            logger.debug('Connected WebSocket')
 
     async def _send_auth(self) -> None:
-        auth_msg = json.dumps({
-            'uid': 0,
-            'roomid': self._room_id,  # must not be the short id!
-            'protover': WS.BODY_PROTOCOL_VERSION_BROTLI,
-            'platform': 'web',
-            'type': 2,
-            'key': self._danmu_info['token'],
-        })
+        auth_msg = json.dumps(
+            {
+                'uid': 0,
+                'roomid': self._room_id,  # must not be the short id!
+                'protover': WS.BODY_PROTOCOL_VERSION_BROTLI,
+                'platform': 'web',
+                'type': 2,
+                'key': self._danmu_info['token'],
+            }
+        )
         data = Frame.encode(WS.OP_USER_AUTHENTICATION, auth_msg)
-        await self._ws.send_bytes(data)
-        logger.debug('Sent user authentication')
+        logger.debug('Sending user authentication...')
+        try:
+            await self._ws.send_bytes(data)
+        except Exception as exc:
+            logger.debug(f'Failed to sent user authentication: {repr(exc)}')
+            raise
+        else:
+            logger.debug('Sent user authentication')
 
     async def _recieve_auth_reply(self) -> aiohttp.WSMessage:
-        msg = await self._ws.receive(timeout=5)
-        if msg.type != aiohttp.WSMsgType.BINARY:
-            raise aiohttp.ClientError(msg)
-        logger.debug('Recieved reply')
-        return msg
+        logger.debug('Receiving user authentication reply...')
+        try:
+            msg = await self._ws.receive(timeout=5)
+            if msg.type != aiohttp.WSMsgType.BINARY:
+                raise aiohttp.ClientError(msg)
+        except Exception as exc:
+            logger.debug(f'Failed to receive user authentication reply: {repr(exc)}')
+            raise
+        else:
+            logger.debug('Recieved user authentication reply')
+            return msg
 
     async def _handle_auth_reply(self, reply: aiohttp.WSMessage) -> None:
         op, msg = Frame.decode(reply.data)
@@ -152,11 +169,18 @@ class DanmakuClient(EventEmitter[DanmakuListener], AsyncStoppableMixin):
         else:
             raise ValueError(f'Unexpected code: {code}')
 
+    def _rotate_api_platform(self) -> None:
+        if self._api_platform == 'android':
+            self._api_platform = 'web'
+        else:
+            self._api_platform = 'android'
+
     async def _update_danmu_info(self) -> None:
-        try:
-            self._danmu_info = await self.appapi.get_danmu_info(self._room_id)
-        except Exception:
+        logger.debug(f'Updating danmu info via {self._api_platform} api...')
+        if self._api_platform == 'web':
             self._danmu_info = await self.webapi.get_danmu_info(self._room_id)
+        else:
+            self._danmu_info = await self.appapi.get_danmu_info(self._room_id)
         logger.debug('Danmu info updated')
 
     async def _disconnect(self) -> None:
@@ -217,7 +241,7 @@ class DanmakuClient(EventEmitter[DanmakuListener], AsyncStoppableMixin):
             wsmsg = await self._ws.receive(timeout=self._HEARTBEAT_INTERVAL)
 
             if wsmsg.type == aiohttp.WSMsgType.BINARY:
-                if (result := await self._handle_data(wsmsg.data)):
+                if result := await self._handle_data(wsmsg.data):
                     return result
             elif wsmsg.type == aiohttp.WSMsgType.ERROR:
                 await self._handle_error(cast(Exception, wsmsg.data))
@@ -249,10 +273,11 @@ class DanmakuClient(EventEmitter[DanmakuListener], AsyncStoppableMixin):
     async def _retry(self) -> None:
         if self._retry_count < self._MAX_RETRIES:
             if self._retry_delay > 0:
-                logger.debug('Retry after {} second{}'.format(
-                    self._retry_delay,
-                    's' if self._retry_delay > 1 else '',
-                ))
+                logger.debug(
+                    'Retry after {} second{}'.format(
+                        self._retry_delay, 's' if self._retry_delay > 1 else ''
+                    )
+                )
                 await asyncio.sleep(self._retry_delay)
             await self.reconnect()
             self._retry_count += 1
@@ -285,9 +310,7 @@ class Frame:
 
     @staticmethod
     def decode(data: bytes) -> Tuple[int, Union[int, str, List[str]]]:
-        plen, hlen, ver, op, _ = struct.unpack_from(
-            Frame.HEADER_FORMAT, data, 0
-        )
+        plen, hlen, ver, op, _ = struct.unpack_from(Frame.HEADER_FORMAT, data, 0)
         body = data[hlen:]
 
         if op == WS.OP_MESSAGE:
@@ -300,7 +323,7 @@ class Frame:
                 plen, hlen, ver, op, _ = struct.unpack_from(
                     Frame.HEADER_FORMAT, data, offset
                 )
-                body = data[hlen + offset:plen + offset]
+                body = data[hlen + offset : plen + offset]
                 msg = body.decode('utf8')
                 msg_list.append(msg)
                 offset += plen

src/blrec/bili/exceptions.py

@@ -34,3 +34,7 @@ class NoStreamCodecAvailable(Exception):
 
 class NoStreamQualityAvailable(Exception):
     pass
+
+
+class NoAlternativeStreamAvailable(Exception):
+    pass

src/blrec/bili/live.py

@@ -1,28 +1,27 @@
-import re
-import json
 import asyncio
+import json
+import re
 from typing import Dict, List, cast
 
 import aiohttp
 from jsonpath import jsonpath
-from tenacity import (
-    retry,
-    wait_exponential,
-    stop_after_delay,
-    retry_if_exception_type,
-)
-
+from tenacity import retry, retry_if_exception_type, stop_after_delay, wait_exponential
 
 from .api import AppApi, WebApi
-from .models import LiveStatus, RoomInfo, UserInfo
-from .typing import StreamFormat, QualityNumber, StreamCodec, ResponseData
 from .exceptions import (
-    LiveRoomHidden, LiveRoomLocked, LiveRoomEncrypted, NoStreamAvailable,
-    NoStreamFormatAvailable, NoStreamCodecAvailable, NoStreamQualityAvailable,
+    LiveRoomEncrypted,
+    LiveRoomHidden,
+    LiveRoomLocked,
+    NoAlternativeStreamAvailable,
+    NoStreamAvailable,
+    NoStreamCodecAvailable,
+    NoStreamFormatAvailable,
+    NoStreamQualityAvailable,
 )
+from .models import LiveStatus, RoomInfo, UserInfo
+from .typing import ApiPlatform, QualityNumber, ResponseData, StreamCodec, StreamFormat
 
-
-__all__ = 'Live',
+__all__ = ('Live',)
 
 
 _INFO_PATTERN = re.compile(
@@ -32,9 +31,7 @@ _LIVE_STATUS_PATTERN = re.compile(rb'"live_status"\s*:\s*(\d)')
 
 
 class Live:
-    def __init__(
-        self, room_id: int, user_agent: str = '', cookie: str = ''
-    ) -> None:
+    def __init__(self, room_id: int, user_agent: str = '', cookie: str = '') -> None:
         self._room_id = room_id
         self._user_agent = user_agent
         self._cookie = cookie
@@ -140,9 +137,7 @@ class Live:
         self._room_info = await self.get_room_info()
 
     @retry(
-        retry=retry_if_exception_type((
-            asyncio.TimeoutError, aiohttp.ClientError,
-        )),
+        retry=retry_if_exception_type((asyncio.TimeoutError, aiohttp.ClientError)),
         wait=wait_exponential(max=10),
         stop=stop_after_delay(60),
     )
@@ -156,56 +151,64 @@ class Live:
         return RoomInfo.from_data(room_info_data)
 
     @retry(
-        retry=retry_if_exception_type((
-            asyncio.TimeoutError, aiohttp.ClientError,
-        )),
+        retry=retry_if_exception_type((asyncio.TimeoutError, aiohttp.ClientError)),
         wait=wait_exponential(max=10),
         stop=stop_after_delay(60),
     )
     async def get_user_info(self, uid: int) -> UserInfo:
         try:
-            user_info_data = await self._appapi.get_user_info(uid)
-            return UserInfo.from_app_api_data(user_info_data)
-        except Exception:
             user_info_data = await self._webapi.get_user_info(uid)
             return UserInfo.from_web_api_data(user_info_data)
+        except Exception:
+            user_info_data = await self._appapi.get_user_info(uid)
+            return UserInfo.from_app_api_data(user_info_data)
 
     async def get_server_timestamp(self) -> int:
         # the timestamp on the server at the moment in seconds
         return await self._webapi.get_timestamp()
 
-    async def get_live_stream_urls(
+    async def get_live_stream_url(
         self,
         qn: QualityNumber = 10000,
+        *,
+        api_platform: ApiPlatform = 'web',
         stream_format: StreamFormat = 'flv',
         stream_codec: StreamCodec = 'avc',
-    ) -> List[str]:
-        try:
-            info = await self._appapi.get_room_play_info(self._room_id, qn)
-        except Exception:
+        select_alternative: bool = False,
+    ) -> str:
+        if api_platform == 'web':
             info = await self._webapi.get_room_play_info(self._room_id, qn)
+        else:
+            info = await self._appapi.get_room_play_info(self._room_id, qn)
 
         self._check_room_play_info(info)
 
         streams = jsonpath(info, '$.playurl_info.playurl.stream[*]')
         if not streams:
-            raise NoStreamAvailable(qn, stream_format, stream_codec)
-        formats = jsonpath(streams, f'$[*].format[?(@.format_name == "{stream_format}")]')  # noqa
+            raise NoStreamAvailable(stream_format, stream_codec, qn)
+        formats = jsonpath(
+            streams, f'$[*].format[?(@.format_name == "{stream_format}")]'
+        )
         if not formats:
-            raise NoStreamFormatAvailable(qn, stream_format, stream_codec)
-        codecs = jsonpath(formats, f'$[*].codec[?(@.codec_name == "{stream_codec}")]')  # noqa
+            raise NoStreamFormatAvailable(stream_format, stream_codec, qn)
+        codecs = jsonpath(formats, f'$[*].codec[?(@.codec_name == "{stream_codec}")]')
         if not codecs:
-            raise NoStreamCodecAvailable(qn, stream_format, stream_codec)
+            raise NoStreamCodecAvailable(stream_format, stream_codec, qn)
         codec = codecs[0]
 
         accept_qn = cast(List[QualityNumber], codec['accept_qn'])
         if qn not in accept_qn or codec['current_qn'] != qn:
-            raise NoStreamQualityAvailable(qn, stream_format, stream_codec)
+            raise NoStreamQualityAvailable(stream_format, stream_codec, qn)
 
-        return [
-            i['host'] + codec['base_url'] + i['extra']
-            for i in codec['url_info']
-        ]
+        urls = [i['host'] + codec['base_url'] + i['extra'] for i in codec['url_info']]
+
+        if not select_alternative:
+            return urls[0]
+
+        try:
+            return urls[1]
+        except IndexError:
+            raise NoAlternativeStreamAvailable(stream_format, stream_codec, qn)
 
     def _check_room_play_info(self, data: ResponseData) -> None:
         if data.get('is_hidden'):
@@ -221,11 +224,11 @@ class Live:
 
     async def _get_room_info_via_api(self) -> ResponseData:
         try:
-            info_data = await self._appapi.get_info_by_room(self._room_id)
+            info_data = await self._webapi.get_info_by_room(self._room_id)
             room_info_data = info_data['room_info']
         except Exception:
             try:
-                info_data = await self._webapi.get_info_by_room(self._room_id)
+                info_data = await self._appapi.get_info_by_room(self._room_id)
                 room_info_data = info_data['room_info']
             except Exception:
                 room_info_data = await self._webapi.get_info(self._room_id)

src/blrec/bili/typing.py

@@ -1,7 +1,10 @@
 from typing import Any, Dict, Literal, Mapping
 
 
-Danmaku = Mapping[str, Any]
+ApiPlatform = Literal[
+    'web',
+    'android',
+]
 
 QualityNumber = Literal[
     20000,  # 4K
@@ -26,3 +29,5 @@ StreamCodec = Literal[
 
 JsonResponse = Dict[str, Any]
 ResponseData = Dict[str, Any]
+
+Danmaku = Mapping[str, Any]

src/blrec/core/base_stream_recorder.py

@@ -1,572 +0,0 @@
-import io
-import os
-import re
-import time
-import asyncio
-import logging
-from abc import ABC, abstractmethod
-from threading import Thread, Event
-from datetime import datetime, timezone, timedelta
-from collections import OrderedDict
-
-from typing import Any, BinaryIO, Dict, Iterator, Optional, Tuple
-
-import aiohttp
-import urllib3
-from tqdm import tqdm
-from rx.subject import Subject
-from rx.core import Observable
-from tenacity import (
-    retry,
-    wait_none,
-    wait_fixed,
-    wait_chain,
-    wait_exponential,
-    stop_after_delay,
-    stop_after_attempt,
-    retry_if_exception_type,
-    TryAgain,
-)
-
-from .. import __version__, __prog__, __github__
-from .stream_remuxer import StreamRemuxer
-from .stream_analyzer import StreamProfile
-from .statistics import StatisticsCalculator
-from ..event.event_emitter import EventListener, EventEmitter
-from ..bili.live import Live
-from ..bili.typing import StreamFormat, QualityNumber
-from ..bili.helpers import get_quality_name
-from ..flv.data_analyser import MetaData
-from ..flv.stream_processor import StreamProcessor, BaseOutputFileManager
-from ..utils.mixins import AsyncCooperationMixin, AsyncStoppableMixin
-from ..path import escape_path
-from ..logging.room_id import aio_task_with_room_id
-from ..bili.exceptions import (
-    NoStreamFormatAvailable, NoStreamCodecAvailable, NoStreamQualityAvailable,
-)
-
-
-__all__ = 'BaseStreamRecorder', 'StreamRecorderEventListener', 'StreamProxy'
-
-
-logger = logging.getLogger(__name__)
-logging.getLogger(urllib3.__name__).setLevel(logging.WARNING)
-
-
-class StreamRecorderEventListener(EventListener):
-    async def on_video_file_created(
-        self, path: str, record_start_time: int
-    ) -> None:
-        ...
-
-    async def on_video_file_completed(self, path: str) -> None:
-        ...
-
-    async def on_stream_recording_stopped(self) -> None:
-        ...
-
-
-class BaseStreamRecorder(
-    EventEmitter[StreamRecorderEventListener],
-    AsyncCooperationMixin,
-    AsyncStoppableMixin,
-    ABC,
-):
-    def __init__(
-        self,
-        live: Live,
-        out_dir: str,
-        path_template: str,
-        *,
-        stream_format: StreamFormat = 'flv',
-        quality_number: QualityNumber = 10000,
-        buffer_size: Optional[int] = None,
-        read_timeout: Optional[int] = None,
-        disconnection_timeout: Optional[int] = None,
-        filesize_limit: int = 0,
-        duration_limit: int = 0,
-    ) -> None:
-        super().__init__()
-
-        self._live = live
-        self._progress_bar: Optional[tqdm] = None
-        self._stream_remuxer: Optional[StreamRemuxer] = None
-        self._stream_processor: Optional[StreamProcessor] = None
-        self._dl_calculator = StatisticsCalculator()
-        self._rec_calculator = StatisticsCalculator()
-        self._file_manager = OutputFileManager(
-            live, out_dir, path_template, buffer_size
-        )
-
-        self._stream_format = stream_format
-        self._quality_number = quality_number
-        self._real_stream_format: Optional[StreamFormat] = None
-        self._real_quality_number: Optional[QualityNumber] = None
-        self._use_candidate_stream: bool = False
-        self.buffer_size = buffer_size or io.DEFAULT_BUFFER_SIZE  # bytes
-        self.read_timeout = read_timeout or 3  # seconds
-        self.disconnection_timeout = disconnection_timeout or 600  # seconds
-
-        self._filesize_limit = filesize_limit or 0
-        self._duration_limit = duration_limit or 0
-
-        self._stream_url: str = ''
-        self._stream_host: str = ''
-        self._stream_profile: StreamProfile = {}
-
-        self._connection_recovered = Event()
-
-        def on_file_created(args: Tuple[str, int]) -> None:
-            logger.info(f"Video file created: '{args[0]}'")
-            self._emit_event('video_file_created', *args)
-
-        def on_file_closed(path: str) -> None:
-            logger.info(f"Video file completed: '{path}'")
-            self._emit_event('video_file_completed', path)
-
-        self._file_manager.file_creates.subscribe(on_file_created)
-        self._file_manager.file_closes.subscribe(on_file_closed)
-
-    @property
-    def stream_url(self) -> str:
-        return self._stream_url
-
-    @property
-    def stream_host(self) -> str:
-        return self._stream_host
-
-    @property
-    def dl_total(self) -> int:
-        return self._dl_calculator.count
-
-    @property
-    def dl_rate(self) -> float:
-        return self._dl_calculator.rate
-
-    @property
-    def rec_elapsed(self) -> float:
-        return self._rec_calculator.elapsed
-
-    @property
-    def rec_total(self) -> int:
-        return self._rec_calculator.count
-
-    @property
-    def rec_rate(self) -> float:
-        return self._rec_calculator.rate
-
-    @property
-    def out_dir(self) -> str:
-        return self._file_manager.out_dir
-
-    @out_dir.setter
-    def out_dir(self, value: str) -> None:
-        self._file_manager.out_dir = value
-
-    @property
-    def path_template(self) -> str:
-        return self._file_manager.path_template
-
-    @path_template.setter
-    def path_template(self, value: str) -> None:
-        self._file_manager.path_template = value
-
-    @property
-    def stream_format(self) -> StreamFormat:
-        return self._stream_format
-
-    @stream_format.setter
-    def stream_format(self, value: StreamFormat) -> None:
-        self._stream_format = value
-        self._real_stream_format = None
-
-    @property
-    def quality_number(self) -> QualityNumber:
-        return self._quality_number
-
-    @quality_number.setter
-    def quality_number(self, value: QualityNumber) -> None:
-        self._quality_number = value
-        self._real_quality_number = None
-
-    @property
-    def real_stream_format(self) -> StreamFormat:
-        return self._real_stream_format or self.stream_format
-
-    @property
-    def real_quality_number(self) -> QualityNumber:
-        return self._real_quality_number or self.quality_number
-
-    @property
-    def filesize_limit(self) -> int:
-        if self._stream_processor is not None:
-            return self._stream_processor.filesize_limit
-        else:
-            return self._filesize_limit
-
-    @filesize_limit.setter
-    def filesize_limit(self, value: int) -> None:
-        self._filesize_limit = value
-        if self._stream_processor is not None:
-            self._stream_processor.filesize_limit = value
-
-    @property
-    def duration_limit(self) -> int:
-        if self._stream_processor is not None:
-            return self._stream_processor.duration_limit
-        else:
-            return self._duration_limit
-
-    @duration_limit.setter
-    def duration_limit(self, value: int) -> None:
-        self._duration_limit = value
-        if self._stream_processor is not None:
-            self._stream_processor.duration_limit = value
-
-    @property
-    def recording_path(self) -> Optional[str]:
-        return self._file_manager.curr_path
-
-    @property
-    def metadata(self) -> Optional[MetaData]:
-        if self._stream_processor is not None:
-            return self._stream_processor.metadata
-        else:
-            return None
-
-    @property
-    def stream_profile(self) -> StreamProfile:
-        return self._stream_profile
-
-    def has_file(self) -> bool:
-        return self._file_manager.has_file()
-
-    def get_files(self) -> Iterator[str]:
-        yield from self._file_manager.get_files()
-
-    def clear_files(self) -> None:
-        self._file_manager.clear_files()
-
-    def can_cut_stream(self) -> bool:
-        if self._stream_processor is None:
-            return False
-        return self._stream_processor.can_cut_stream()
-
-    def cut_stream(self) -> bool:
-        if self._stream_processor is None:
-            return False
-        return self._stream_processor.cut_stream()
-
-    def update_progress_bar_info(self) -> None:
-        if self._progress_bar is not None:
-            self._progress_bar.set_postfix_str(self._make_pbar_postfix())
-
-    async def _do_start(self) -> None:
-        logger.debug('Starting stream recorder...')
-        self._dl_calculator.reset()
-        self._rec_calculator.reset()
-        self._stream_url = ''
-        self._stream_host = ''
-        self._stream_profile = {}
-        self._use_candidate_stream = False
-        self._connection_recovered.clear()
-        self._thread = Thread(
-            target=self._run, name=f'StreamRecorder::{self._live.room_id}'
-        )
-        self._thread.start()
-        logger.debug('Started stream recorder')
-
-    async def _do_stop(self) -> None:
-        logger.debug('Stopping stream recorder...')
-        if self._stream_processor is not None:
-            self._stream_processor.cancel()
-        await self._loop.run_in_executor(None, self._thread.join)
-        logger.debug('Stopped stream recorder')
-
-    @abstractmethod
-    def _run(self) -> None:
-        raise NotImplementedError()
-
-    @retry(
-        reraise=True,
-        retry=retry_if_exception_type((
-            asyncio.TimeoutError, aiohttp.ClientError,
-        )),
-        wait=wait_chain(wait_none(), wait_fixed(1)),
-        stop=stop_after_attempt(300),
-    )
-    def _get_live_stream_url(self) -> str:
-        qn = self._real_quality_number or self.quality_number
-        fmt = self._real_stream_format or self.stream_format
-        logger.info(
-            f'Getting the live stream url... qn: {qn}, format: {fmt}, '
-            f'use_candidate_stream: {self._use_candidate_stream}'
-        )
-        try:
-            urls = self._run_coroutine(
-                self._live.get_live_stream_urls(qn, fmt)
-            )
-        except NoStreamQualityAvailable:
-            logger.info(
-                f'The specified stream quality ({qn}) is not available, '
-                'will using the original stream quality (10000) instead.'
-            )
-            self._real_quality_number = 10000
-            raise TryAgain
-        except NoStreamFormatAvailable:
-            if fmt == 'fmp4':
-                logger.info(
-                    'The specified stream format (fmp4) is not available, '
-                    'falling back to stream format (ts).'
-                )
-                self._real_stream_format = 'ts'
-            elif fmt == 'ts':
-                logger.info(
-                    'The specified stream format (ts) is not available, '
-                    'falling back to stream format (flv).'
-                )
-                self._real_stream_format = 'flv'
-            else:
-                raise NotImplementedError(fmt)
-            raise TryAgain
-        except NoStreamCodecAvailable as e:
-            logger.warning(repr(e))
-            raise TryAgain
-        else:
-            logger.info(
-                f'Adopted the stream format ({fmt}) and quality ({qn})'
-            )
-            self._real_quality_number = qn
-            self._real_stream_format = fmt
-
-        if not self._use_candidate_stream:
-            url = urls[0]
-        else:
-            try:
-                url = urls[1]
-            except IndexError:
-                logger.info(
-                    'No candidate stream url available, '
-                    'will using the primary stream url instead.'
-                )
-                url = urls[0]
-        logger.info(f"Got live stream url: '{url}'")
-
-        return url
-
-    def _defer_retry(self, seconds: float, name: str = '') -> None:
-        if seconds <= 0:
-            return
-        logger.debug(f'Retry {name} after {seconds} seconds')
-        time.sleep(seconds)
-
-    def _wait_for_connection_error(self) -> None:
-        Thread(
-            target=self._conectivity_checker,
-            name=f'ConectivityChecker::{self._live.room_id}',
-            daemon=True,
-        ).start()
-        self._connection_recovered.wait()
-        self._connection_recovered.clear()
-
-    def _conectivity_checker(self, check_interval: int = 3) -> None:
-        timeout = self.disconnection_timeout
-        logger.info(f'Waiting {timeout} seconds for connection recovery... ')
-        timebase = time.monotonic()
-        while not self._run_coroutine(self._live.check_connectivity()):
-            if timeout is not None and time.monotonic() - timebase > timeout:
-                logger.error(f'Connection not recovered in {timeout} seconds')
-                self._stopped = True
-                self._connection_recovered.set()
-            time.sleep(check_interval)
-        else:
-            logger.info('Connection recovered')
-            self._connection_recovered.set()
-
-    def _make_pbar_postfix(self) -> str:
-        return '{room_id} - {user_name}: {room_title}'.format(
-            room_id=self._live.room_info.room_id,
-            user_name=self._live.user_info.name,
-            room_title=self._live.room_info.title,
-        )
-
-    def _make_metadata(self) -> Dict[str, Any]:
-        live_start_time = datetime.fromtimestamp(
-            self._live.room_info.live_start_time, timezone(timedelta(hours=8))
-        )
-
-        assert self._real_quality_number is not None
-        stream_quality = '{} ({}{})'.format(
-            get_quality_name(self._real_quality_number),
-            self._real_quality_number,
-            ', bluray' if '_bluray' in self._stream_url else '',
-        )
-
-        return {
-            'Title': self._live.room_info.title,
-            'Artist': self._live.user_info.name,
-            'Date': str(live_start_time),
-            'Comment': f'''\
-B站直播录像
-主播：{self._live.user_info.name}
-标题：{self._live.room_info.title}
-分区：{self._live.room_info.parent_area_name} - {self._live.room_info.area_name}
-房间号：{self._live.room_info.room_id}
-开播时间：{live_start_time}
-流主机: {self._stream_host}
-流格式：{self._real_stream_format}
-流画质：{stream_quality}
-录制程序：{__prog__} v{__version__} {__github__}''',
-            'description': OrderedDict({
-                'UserId': str(self._live.user_info.uid),
-                'UserName': self._live.user_info.name,
-                'RoomId': str(self._live.room_info.room_id),
-                'RoomTitle': self._live.room_info.title,
-                'Area': self._live.room_info.area_name,
-                'ParentArea': self._live.room_info.parent_area_name,
-                'LiveStartTime': str(live_start_time),
-                'StreamHost': self._stream_host,
-                'StreamFormat': self._real_stream_format,
-                'StreamQuality': stream_quality,
-                'Recorder': f'{__prog__} v{__version__} {__github__}',
-            })
-        }
-
-    def _emit_event(self, name: str, *args: Any, **kwds: Any) -> None:
-        self._run_coroutine(self._emit(name, *args, **kwds))
-
-    @aio_task_with_room_id
-    async def _emit(self, *args: Any, **kwds: Any) -> None:  # type: ignore
-        await super()._emit(*args, **kwds)
-
-
-class StreamProxy(io.RawIOBase):
-    def __init__(self, stream: io.BufferedIOBase) -> None:
-        self._stream = stream
-        self._offset = 0
-        self._size_updates = Subject()
-
-    @property
-    def size_updates(self) -> Observable:
-        return self._size_updates
-
-    @property
-    def closed(self) -> bool:
-        # always return False to avoid that `ValueError: read of closed file`,
-        # raised from `CHECK_CLOSED(self, "read of closed file")`,
-        # result in losing data those remaining in the buffer.
-        # ref: `https://gihub.com/python/cpython/blob/63298930fb531ba2bb4f23bc3b915dbf1e17e9e1/Modules/_io/bufferedio.c#L882`  # noqa
-        return False
-
-    def fileno(self) -> int:
-        return self._stream.fileno()
-
-    def readable(self) -> bool:
-        return True
-
-    def read(self, size: int = -1) -> bytes:
-        data = self._stream.read(size)
-        self._offset += len(data)
-        self._size_updates.on_next(len(data))
-        return data
-
-    def tell(self) -> int:
-        return self._offset
-
-    def readinto(self, b: Any) -> int:
-        n = self._stream.readinto(b)
-        self._offset += n
-        self._size_updates.on_next(n)
-        return n
-
-    def close(self) -> None:
-        self._stream.close()
-
-
-class OutputFileManager(BaseOutputFileManager, AsyncCooperationMixin):
-    def __init__(
-        self,
-        live: Live,
-        out_dir: str,
-        path_template: str,
-        buffer_size: Optional[int] = None,
-    ) -> None:
-        super().__init__(buffer_size)
-        self._live = live
-
-        self.out_dir = out_dir
-        self.path_template = path_template
-
-        self._file_creates = Subject()
-        self._file_closes = Subject()
-
-    @property
-    def file_creates(self) -> Observable:
-        return self._file_creates
-
-    @property
-    def file_closes(self) -> Observable:
-        return self._file_closes
-
-    def create_file(self) -> BinaryIO:
-        self._start_time = self._get_timestamp()
-        file = super().create_file()
-        self._file_creates.on_next((self._curr_path, self._start_time))
-        return file
-
-    def close_file(self) -> None:
-        path = self._curr_path
-        super().close_file()
-        self._file_closes.on_next(path)
-
-    def _get_timestamp(self) -> int:
-        try:
-            return self._get_server_timestamp()
-        except Exception as e:
-            logger.warning(f'Failed to get server timestamp: {repr(e)}')
-            return self._get_local_timestamp()
-
-    def _get_local_timestamp(self) -> int:
-        return int(time.time())
-
-    @retry(
-        reraise=True,
-        retry=retry_if_exception_type((
-            asyncio.TimeoutError, aiohttp.ClientError,
-        )),
-        wait=wait_exponential(multiplier=0.1, max=1),
-        stop=stop_after_delay(3),
-    )
-    def _get_server_timestamp(self) -> int:
-        return self._run_coroutine(self._live.get_server_timestamp())
-
-    def _make_path(self) -> str:
-        date_time = datetime.fromtimestamp(self._start_time)
-        relpath = self.path_template.format(
-            roomid=self._live.room_id,
-            uname=escape_path(self._live.user_info.name),
-            title=escape_path(self._live.room_info.title),
-            area=escape_path(self._live.room_info.area_name),
-            parent_area=escape_path(self._live.room_info.parent_area_name),
-            year=date_time.year,
-            month=str(date_time.month).rjust(2, '0'),
-            day=str(date_time.day).rjust(2, '0'),
-            hour=str(date_time.hour).rjust(2, '0'),
-            minute=str(date_time.minute).rjust(2, '0'),
-            second=str(date_time.second).rjust(2, '0'),
-        )
-
-        pathname = os.path.abspath(
-            os.path.expanduser(os.path.join(self.out_dir, relpath) + '.flv')
-        )
-        os.makedirs(os.path.dirname(pathname), exist_ok=True)
-        while os.path.exists(pathname):
-            root, ext = os.path.splitext(pathname)
-            m = re.search(r'_\((\d+)\)$', root)
-            if m is None:
-                root += '_(1)'
-            else:
-                root = re.sub(r'\(\d+\)$', f'({int(m.group(1)) + 1})', root)
-            pathname = root + ext
-
-        return pathname

src/blrec/core/cover_downloader.py

@@ -0,0 +1,97 @@
+import asyncio
+import logging
+from enum import Enum
+from typing import Set
+
+import aiofiles
+import aiohttp
+from tenacity import retry, stop_after_attempt, wait_fixed
+
+from ..bili.live import Live
+from ..exception import exception_callback
+from ..logging.room_id import aio_task_with_room_id
+from ..path import cover_path
+from ..utils.hash import sha1sum
+from ..utils.mixins import SwitchableMixin
+from .stream_recorder import StreamRecorder, StreamRecorderEventListener
+
+__all__ = ('CoverDownloader',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class CoverSaveStrategy(Enum):
+    DEFAULT = 'default'
+    DEDUP = 'dedup'
+
+    def __str__(self) -> str:
+        return self.value
+
+    # workaround for value serialization
+    def __repr__(self) -> str:
+        return str(self)
+
+
+class CoverDownloader(StreamRecorderEventListener, SwitchableMixin):
+    def __init__(
+        self,
+        live: Live,
+        stream_recorder: StreamRecorder,
+        *,
+        save_cover: bool = False,
+        cover_save_strategy: CoverSaveStrategy = CoverSaveStrategy.DEFAULT,
+    ) -> None:
+        super().__init__()
+        self._live = live
+        self._stream_recorder = stream_recorder
+        self._lock: asyncio.Lock = asyncio.Lock()
+        self._sha1_set: Set[str] = set()
+        self.save_cover = save_cover
+        self.cover_save_strategy = cover_save_strategy
+
+    def _do_enable(self) -> None:
+        self._sha1_set.clear()
+        self._stream_recorder.add_listener(self)
+        logger.debug('Enabled cover downloader')
+
+    def _do_disable(self) -> None:
+        self._stream_recorder.remove_listener(self)
+        logger.debug('Disabled cover downloader')
+
+    async def on_video_file_completed(self, video_path: str) -> None:
+        async with self._lock:
+            if not self.save_cover:
+                return
+            task = asyncio.create_task(self._save_cover(video_path))
+            task.add_done_callback(exception_callback)
+
+    @aio_task_with_room_id
+    async def _save_cover(self, video_path: str) -> None:
+        try:
+            await self._live.update_info()
+            cover_url = self._live.room_info.cover
+            data = await self._fetch_cover(cover_url)
+            sha1 = sha1sum(data)
+            if (
+                self.cover_save_strategy == CoverSaveStrategy.DEDUP
+                and sha1 in self._sha1_set
+            ):
+                return
+            path = cover_path(video_path, ext=cover_url.rsplit('.', 1)[-1])
+            await self._save_file(path, data)
+            self._sha1_set.add(sha1)
+        except Exception as e:
+            logger.error(f'Failed to save cover image: {repr(e)}')
+        else:
+            logger.info(f'Saved cover image: {path}')
+
+    @retry(reraise=True, wait=wait_fixed(1), stop=stop_after_attempt(3))
+    async def _fetch_cover(self, url: str) -> bytes:
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            async with session.get(url) as response:
+                return await response.read()
+
+    async def _save_file(self, path: str, data: bytes) -> None:
+        async with aiofiles.open(path, 'wb') as file:
+            await file.write(data)

src/blrec/core/danmaku_dumper.py

@@ -12,10 +12,8 @@ from tenacity import (
 
 from .. import __version__, __prog__, __github__
 from .danmaku_receiver import DanmakuReceiver, DanmuMsg
-from .base_stream_recorder import (
-    BaseStreamRecorder, StreamRecorderEventListener
-)
-from .statistics import StatisticsCalculator
+from .stream_recorder import StreamRecorder, StreamRecorderEventListener
+from .statistics import Statistics
 from ..bili.live import Live
 from ..exception import exception_callback, submit_exception
 from ..event.event_emitter import EventListener, EventEmitter
@@ -51,7 +49,7 @@ class DanmakuDumper(
     def __init__(
         self,
         live: Live,
-        stream_recorder: BaseStreamRecorder,
+        stream_recorder: StreamRecorder,
         danmaku_receiver: DanmakuReceiver,
         *,
         danmu_uname: bool = False,
@@ -72,34 +70,27 @@ class DanmakuDumper(
         self.record_guard_buy = record_guard_buy
         self.record_super_chat = record_super_chat
 
+        self._lock: asyncio.Lock = asyncio.Lock()
         self._path: Optional[str] = None
         self._files: List[str] = []
-        self._calculator = StatisticsCalculator(interval=60)
+        self._statistics = Statistics(interval=60)
 
     @property
     def danmu_total(self) -> int:
-        return self._calculator.count
+        return self._statistics.count
 
     @property
     def danmu_rate(self) -> float:
-        return self._calculator.rate
+        return self._statistics.rate
 
     @property
     def elapsed(self) -> float:
-        return self._calculator.elapsed
+        return self._statistics.elapsed
 
     @property
     def dumping_path(self) -> Optional[str]:
         return self._path
 
-    def change_stream_recorder(
-        self, stream_recorder: BaseStreamRecorder
-    ) -> None:
-        self._stream_recorder.remove_listener(self)
-        self._stream_recorder = stream_recorder
-        self._stream_recorder.add_listener(self)
-        logger.debug('Changed stream recorder')
-
     def _do_enable(self) -> None:
         self._stream_recorder.add_listener(self)
         logger.debug('Enabled danmaku dumper')
@@ -124,14 +115,16 @@ class DanmakuDumper(
     async def on_video_file_created(
         self, video_path: str, record_start_time: int
     ) -> None:
-        self._path = danmaku_path(video_path)
-        self._record_start_time = record_start_time
-        self._files.append(self._path)
-        self._start_dumping()
+        async with self._lock:
+            self._path = danmaku_path(video_path)
+            self._record_start_time = record_start_time
+            self._files.append(self._path)
+            self._start_dumping()
 
     async def on_video_file_completed(self, video_path: str) -> None:
-        await self._stop_dumping()
-        self._path = None
+        async with self._lock:
+            await self._stop_dumping()
+            self._path = None
 
     def _start_dumping(self) -> None:
         self._create_dump_task()
@@ -152,7 +145,7 @@ class DanmakuDumper(
     async def _do_dump(self) -> None:
         assert self._path is not None
         logger.debug('Started dumping danmaku')
-        self._calculator.reset()
+        self._statistics.reset()
 
         try:
             async with DanmakuWriter(self._path) as writer:
@@ -176,14 +169,14 @@ class DanmakuDumper(
             logger.info(f"Danmaku file completed: '{self._path}'")
             await self._emit('danmaku_file_completed', self._path)
             logger.debug('Stopped dumping danmaku')
-            self._calculator.freeze()
+            self._statistics.freeze()
 
     async def _dumping_loop(self, writer: DanmakuWriter) -> None:
         while True:
             msg = await self._receiver.get_message()
             if isinstance(msg, DanmuMsg):
                 await writer.write_danmu(self._make_danmu(msg))
-                self._calculator.submit(1)
+                self._statistics.submit(1)
             elif isinstance(msg, GiftSendMsg):
                 if not self.record_gift_send:
                     continue

src/blrec/core/flv_stream_recorder.py

@@ -1,214 +0,0 @@
-import io
-import errno
-import logging
-from urllib.parse import urlparse
-
-from typing import Optional
-
-import urllib3
-import requests
-from tqdm import tqdm
-from tenacity import (
-    retry_if_result,
-    retry_if_not_exception_type,
-    Retrying,
-    TryAgain,
-)
-
-from .stream_analyzer import StreamProfile
-from .base_stream_recorder import BaseStreamRecorder, StreamProxy
-from .retry import wait_exponential_for_same_exceptions, before_sleep_log
-from ..bili.live import Live
-from ..bili.typing import StreamFormat, QualityNumber
-from ..flv.stream_processor import StreamProcessor
-from ..utils.mixins import AsyncCooperationMixin, AsyncStoppableMixin
-from ..flv.exceptions import FlvDataError, FlvStreamCorruptedError
-from ..bili.exceptions import (
-    LiveRoomHidden, LiveRoomLocked, LiveRoomEncrypted, NoStreamAvailable,
-)
-
-
-__all__ = 'FLVStreamRecorder',
-
-
-logger = logging.getLogger(__name__)
-
-
-class FLVStreamRecorder(
-    BaseStreamRecorder,
-    AsyncCooperationMixin,
-    AsyncStoppableMixin,
-):
-    def __init__(
-        self,
-        live: Live,
-        out_dir: str,
-        path_template: str,
-        *,
-        stream_format: StreamFormat = 'flv',
-        quality_number: QualityNumber = 10000,
-        buffer_size: Optional[int] = None,
-        read_timeout: Optional[int] = None,
-        disconnection_timeout: Optional[int] = None,
-        filesize_limit: int = 0,
-        duration_limit: int = 0,
-    ) -> None:
-        super().__init__(
-            live=live,
-            out_dir=out_dir,
-            path_template=path_template,
-            stream_format=stream_format,
-            quality_number=quality_number,
-            buffer_size=buffer_size,
-            read_timeout=read_timeout,
-            disconnection_timeout=disconnection_timeout,
-            filesize_limit=filesize_limit,
-            duration_limit=duration_limit,
-        )
-
-    def _run(self) -> None:
-        logger.debug('Stream recorder thread started')
-        try:
-            with tqdm(
-                desc='Recording',
-                unit='B',
-                unit_scale=True,
-                postfix=self._make_pbar_postfix(),
-            ) as progress_bar:
-                self._progress_bar = progress_bar
-
-                self._stream_processor = StreamProcessor(
-                    self._file_manager,
-                    filesize_limit=self._filesize_limit,
-                    duration_limit=self._duration_limit,
-                    analyse_data=True,
-                    dedup_join=True,
-                    save_extra_metadata=True,
-                )
-
-                def update_size(size: int) -> None:
-                    progress_bar.update(size)
-                    self._rec_calculator.submit(size)
-
-                def update_stream_profile(profile: StreamProfile) -> None:
-                    self._stream_profile = profile
-
-                self._stream_processor.size_updates.subscribe(update_size)
-                self._stream_processor.stream_profile_updates.subscribe(
-                    update_stream_profile
-                )
-
-                with requests.Session() as self._session:
-                    self._main_loop()
-        except TryAgain:
-            pass
-        except Exception as e:
-            self._handle_exception(e)
-        finally:
-            if self._stream_processor is not None:
-                self._stream_processor.finalize()
-                self._stream_processor = None
-            self._progress_bar = None
-            self._dl_calculator.freeze()
-            self._rec_calculator.freeze()
-            self._emit_event('stream_recording_stopped')
-            logger.debug('Stream recorder thread stopped')
-
-    def _main_loop(self) -> None:
-        for attempt in Retrying(
-            reraise=True,
-            retry=(
-                retry_if_result(lambda r: not self._stopped) |
-                retry_if_not_exception_type((OSError, NotImplementedError))
-            ),
-            wait=wait_exponential_for_same_exceptions(max=60),
-            before_sleep=before_sleep_log(logger, logging.DEBUG, 'main_loop'),
-        ):
-            with attempt:
-                try:
-                    self._streaming_loop()
-                except NoStreamAvailable as e:
-                    logger.warning(f'No stream available: {repr(e)}')
-                    if not self._stopped:
-                        raise TryAgain
-                except OSError as e:
-                    logger.critical(repr(e), exc_info=e)
-                    if e.errno == errno.ENOSPC:
-                        # OSError(28, 'No space left on device')
-                        self._handle_exception(e)
-                        self._stopped = True
-                    raise TryAgain
-                except LiveRoomHidden:
-                    logger.error('The live room has been hidden!')
-                    self._stopped = True
-                except LiveRoomLocked:
-                    logger.error('The live room has been locked!')
-                    self._stopped = True
-                except LiveRoomEncrypted:
-                    logger.error('The live room has been encrypted!')
-                    self._stopped = True
-                except Exception as e:
-                    logger.exception(e)
-                    self._handle_exception(e)
-                    self._stopped = True
-
-    def _streaming_loop(self) -> None:
-        url = self._get_live_stream_url()
-
-        while not self._stopped:
-            try:
-                self._streaming(url)
-            except requests.exceptions.HTTPError as e:
-                # frequently occurred when the live just started or ended.
-                logger.warning(repr(e))
-                self._defer_retry(1, 'streaming_loop')
-                # the url may has been forbidden or expired
-                # when the status code is 404 or 403
-                if e.response.status_code in (403, 404):
-                    url = self._get_live_stream_url()
-            except requests.exceptions.Timeout as e:
-                logger.warning(repr(e))
-            except urllib3.exceptions.TimeoutError as e:
-                logger.warning(repr(e))
-            except urllib3.exceptions.ProtocolError as e:
-                # ProtocolError('Connection broken: IncompleteRead(
-                logger.warning(repr(e))
-            except requests.exceptions.ConnectionError as e:
-                logger.warning(repr(e))
-                self._wait_for_connection_error()
-            except FlvDataError as e:
-                logger.warning(repr(e))
-                self._use_candidate_stream = not self._use_candidate_stream
-                url = self._get_live_stream_url()
-            except FlvStreamCorruptedError as e:
-                logger.warning(repr(e))
-                url = self._get_live_stream_url()
-
-    def _streaming(self, url: str) -> None:
-        logger.debug(f'Requesting live stream... {url}')
-        self._stream_url = url
-        self._stream_host = urlparse(url).hostname or ''
-
-        with self._session.get(
-            url,
-            stream=True,
-            headers=self._live.headers,
-            timeout=self.read_timeout,
-        ) as response:
-            logger.debug('Response received')
-            response.raise_for_status()
-
-            if self._stopped:
-                return
-
-            assert self._stream_processor is not None
-            self._stream_processor.set_metadata(self._make_metadata())
-
-            stream_proxy = StreamProxy(response.raw)
-            stream_proxy.size_updates.subscribe(
-                lambda n: self._dl_calculator.submit(n)
-            )
-
-            self._stream_processor.process_stream(
-                io.BufferedReader(stream_proxy, buffer_size=8192)
-            )

src/blrec/core/flv_stream_recorder_impl.py

@@ -0,0 +1,70 @@
+import logging
+from typing import Optional
+
+from reactivex.scheduler import NewThreadScheduler
+
+from ..bili.live import Live
+from ..bili.typing import QualityNumber
+from ..flv import operators as flv_ops
+from .stream_recorder_impl import StreamRecorderImpl
+
+__all__ = ('FLVStreamRecorderImpl',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class FLVStreamRecorderImpl(StreamRecorderImpl):
+    def __init__(
+        self,
+        live: Live,
+        out_dir: str,
+        path_template: str,
+        *,
+        quality_number: QualityNumber = 10000,
+        buffer_size: Optional[int] = None,
+        read_timeout: Optional[int] = None,
+        disconnection_timeout: Optional[int] = None,
+        filesize_limit: int = 0,
+        duration_limit: int = 0,
+    ) -> None:
+        super().__init__(
+            live=live,
+            out_dir=out_dir,
+            path_template=path_template,
+            stream_format='flv',
+            quality_number=quality_number,
+            buffer_size=buffer_size,
+            read_timeout=read_timeout,
+            disconnection_timeout=disconnection_timeout,
+            filesize_limit=filesize_limit,
+            duration_limit=duration_limit,
+        )
+
+    def _run(self) -> None:
+        self._subscription = (
+            self._stream_param_holder.get_stream_params()  # type: ignore
+            .pipe(
+                self._stream_url_resolver,
+                self._stream_fetcher,
+                self._dl_statistics,
+                self._stream_parser,
+                self._request_exception_handler,
+                self._connection_error_handler,
+                flv_ops.process(),
+                self._cutter,
+                self._limiter,
+                self._join_point_extractor,
+                self._prober,
+                self._injector,
+                self._analyser,
+                self._dumper,
+                self._rec_statistics,
+                self._progress_bar,
+                self._exception_handler,
+            )
+            .subscribe(
+                on_completed=lambda: self._emit_event('stream_recording_completed'),
+                scheduler=NewThreadScheduler(self._thread_factory('StreamRecorder')),
+            )
+        )

src/blrec/core/hls_stream_recorder.py

@@ -1,442 +0,0 @@
-import io
-import time
-import errno
-import logging
-from queue import Queue, Empty
-from threading import Thread, Event, Lock
-from datetime import datetime
-from contextlib import suppress
-from urllib.parse import urlparse
-
-from typing import Set, Optional
-
-import urllib3
-import requests
-import m3u8
-from m3u8.model import Segment
-from tqdm import tqdm
-from tenacity import (
-    retry,
-    wait_exponential,
-    stop_after_delay,
-    retry_if_result,
-    retry_if_exception_type,
-    retry_if_not_exception_type,
-    Retrying,
-    TryAgain,
-    RetryError,
-)
-
-from .stream_remuxer import StreamRemuxer
-from .stream_analyzer import ffprobe, StreamProfile
-from .base_stream_recorder import BaseStreamRecorder, StreamProxy
-from .retry import wait_exponential_for_same_exceptions, before_sleep_log
-from ..bili.live import Live
-from ..bili.typing import StreamFormat, QualityNumber
-from ..flv.stream_processor import StreamProcessor
-from ..utils.mixins import (
-    AsyncCooperationMixin, AsyncStoppableMixin, SupportDebugMixin
-)
-from ..bili.exceptions import (
-    LiveRoomHidden, LiveRoomLocked, LiveRoomEncrypted, NoStreamAvailable,
-)
-
-
-__all__ = 'HLSStreamRecorder',
-
-
-logger = logging.getLogger(__name__)
-
-
-class HLSStreamRecorder(
-    BaseStreamRecorder,
-    AsyncCooperationMixin,
-    AsyncStoppableMixin,
-    SupportDebugMixin,
-):
-    def __init__(
-        self,
-        live: Live,
-        out_dir: str,
-        path_template: str,
-        *,
-        stream_format: StreamFormat = 'flv',
-        quality_number: QualityNumber = 10000,
-        buffer_size: Optional[int] = None,
-        read_timeout: Optional[int] = None,
-        disconnection_timeout: Optional[int] = None,
-        filesize_limit: int = 0,
-        duration_limit: int = 0,
-    ) -> None:
-        super().__init__(
-            live=live,
-            out_dir=out_dir,
-            path_template=path_template,
-            stream_format=stream_format,
-            quality_number=quality_number,
-            buffer_size=buffer_size,
-            read_timeout=read_timeout,
-            disconnection_timeout=disconnection_timeout,
-            filesize_limit=filesize_limit,
-            duration_limit=duration_limit,
-        )
-        self._init_for_debug(self._live.room_id)
-        self._stream_analysed_lock = Lock()
-        self._last_segment_uris: Set[str] = set()
-
-    def _run(self) -> None:
-        logger.debug('Stream recorder thread started')
-        try:
-            if self._debug:
-                path = '{}/playlist-{}-{}.m3u8'.format(
-                    self._debug_dir,
-                    self._live.room_id,
-                    datetime.now().strftime('%Y-%m-%d-%H%M%S-%f'),
-                )
-                self._playlist_debug_file = open(path, 'wt', encoding='utf-8')
-
-            with StreamRemuxer(self._live.room_id) as self._stream_remuxer:
-                with requests.Session() as self._session:
-                    self._session.headers.update(self._live.headers)
-
-                    self._segment_queue: Queue[Segment] = Queue(maxsize=1000)
-                    self._segment_data_queue: Queue[bytes] = Queue(maxsize=100)
-                    self._stream_host_available = Event()
-
-                    self._segment_fetcher_thread = Thread(
-                        target=self._run_segment_fetcher,
-                        name=f'SegmentFetcher::{self._live.room_id}',
-                        daemon=True,
-                    )
-                    self._segment_fetcher_thread.start()
-
-                    self._segment_data_feeder_thread = Thread(
-                        target=self._run_segment_data_feeder,
-                        name=f'SegmentDataFeeder::{self._live.room_id}',
-                        daemon=True,
-                    )
-                    self._segment_data_feeder_thread.start()
-
-                    self._stream_processor_thread = Thread(
-                        target=self._run_stream_processor,
-                        name=f'StreamProcessor::{self._live.room_id}',
-                        daemon=True,
-                    )
-                    self._stream_processor_thread.start()
-
-                    try:
-                        self._main_loop()
-                    finally:
-                        if self._stream_processor is not None:
-                            self._stream_processor.cancel()
-                        self._segment_fetcher_thread.join(timeout=10)
-                        self._segment_data_feeder_thread.join(timeout=10)
-                        self._last_segment_uris.clear()
-                        del self._segment_queue
-                        del self._segment_data_queue
-        except TryAgain:
-            pass
-        except Exception as e:
-            self._handle_exception(e)
-        finally:
-            with suppress(Exception):
-                self._stream_processor_thread.join(timeout=10)
-            with suppress(Exception):
-                self._playlist_debug_file.close()
-            self._emit_event('stream_recording_stopped')
-            logger.debug('Stream recorder thread stopped')
-
-    def _main_loop(self) -> None:
-        for attempt in Retrying(
-            reraise=True,
-            retry=(
-                retry_if_result(lambda r: not self._stopped) |
-                retry_if_not_exception_type((OSError, NotImplementedError))
-            ),
-            wait=wait_exponential_for_same_exceptions(max=60),
-            before_sleep=before_sleep_log(logger, logging.DEBUG, 'main_loop'),
-        ):
-            with attempt:
-                try:
-                    self._streaming_loop()
-                except NoStreamAvailable as e:
-                    logger.warning(f'No stream available: {repr(e)}')
-                    if not self._stopped:
-                        raise TryAgain
-                except OSError as e:
-                    logger.critical(repr(e), exc_info=e)
-                    if e.errno == errno.ENOSPC:
-                        # OSError(28, 'No space left on device')
-                        self._handle_exception(e)
-                        self._stopped = True
-                    raise TryAgain
-                except LiveRoomHidden:
-                    logger.error('The live room has been hidden!')
-                    self._stopped = True
-                except LiveRoomLocked:
-                    logger.error('The live room has been locked!')
-                    self._stopped = True
-                except LiveRoomEncrypted:
-                    logger.error('The live room has been encrypted!')
-                    self._stopped = True
-                except Exception as e:
-                    logger.exception(e)
-                    self._handle_exception(e)
-                    self._stopped = True
-
-    def _streaming_loop(self) -> None:
-        url = self._get_live_stream_url()
-
-        while not self._stopped:
-            try:
-                self._playlist_fetcher(url)
-            except requests.exceptions.HTTPError as e:
-                # frequently occurred when the live just started or ended.
-                logger.warning(repr(e))
-                self._defer_retry(1, 'streaming_loop')
-                # the url may has been forbidden or expired
-                # when the status code is 404 or 403
-                if e.response.status_code in (403, 404):
-                    url = self._get_live_stream_url()
-            except requests.exceptions.ConnectionError as e:
-                logger.warning(repr(e))
-                self._wait_for_connection_error()
-            except RetryError as e:
-                logger.warning(repr(e))
-
-    def _playlist_fetcher(self, url: str) -> None:
-        self._stream_url = url
-        self._stream_host = urlparse(url).hostname or ''
-        self._stream_host_available.set()
-        with self._stream_analysed_lock:
-            self._stream_analysed = False
-
-        while not self._stopped:
-            content = self._fetch_playlist(url)
-            playlist = m3u8.loads(content, uri=url)
-
-            if self._debug:
-                self._playlist_debug_file.write(content + '\n')
-
-            if playlist.is_variant:
-                url = sorted(
-                    playlist.playlists,
-                    key=lambda p: p.stream_info.bandwidth
-                )[-1].absolute_uri
-                logger.debug(f'playlist changed to variant playlist: {url}')
-                self._stream_url = url
-                self._stream_host = urlparse(url).hostname or ''
-                with self._stream_analysed_lock:
-                    self._stream_analysed = False
-                continue
-
-            uris: Set[str] = set()
-            for seg in playlist.segments:
-                uris.add(seg.uri)
-                if seg.uri not in self._last_segment_uris:
-                    self._segment_queue.put(seg, timeout=60)
-
-            if (
-                self._last_segment_uris and
-                not uris.intersection(self._last_segment_uris)
-            ):
-                logger.debug(
-                    'segments broken!\n'
-                    f'last segments: {self._last_segment_uris}\n'
-                    f'current segments: {uris}'
-                )
-                with self._stream_analysed_lock:
-                    self._stream_analysed = False
-
-            self._last_segment_uris = uris
-
-            if playlist.is_endlist:
-                logger.debug('playlist ended')
-                self._stopped = True
-                break
-
-            time.sleep(1)
-
-    def _run_segment_fetcher(self) -> None:
-        logger.debug('Segment fetcher thread started')
-        try:
-            self._segment_fetcher()
-        except Exception as e:
-            logger.exception(e)
-            self._handle_exception(e)
-        finally:
-            self._dl_calculator.freeze()
-            logger.debug('Segment fetcher thread stopped')
-
-    def _segment_fetcher(self) -> None:
-        assert self._stream_remuxer is not None
-        init_section = None
-        self._init_section_data = None
-
-        while not self._stopped:
-            try:
-                seg = self._segment_queue.get(timeout=1)
-            except Empty:
-                continue
-            for attempt in Retrying(
-                reraise=True,
-                retry=(
-                    retry_if_result(lambda r: not self._stopped) |
-                    retry_if_not_exception_type((OSError, NotImplementedError))
-                ),
-            ):
-                if attempt.retry_state.attempt_number > 3:
-                    break
-                with attempt:
-                    try:
-                        if (
-                            getattr(seg, 'init_section', None) and
-                            (
-                                not init_section or
-                                seg.init_section.uri != init_section.uri
-                            )
-                        ):
-                            data = self._fetch_segment(
-                                seg.init_section.absolute_uri
-                            )
-                            init_section = seg.init_section
-                            self._init_section_data = data
-                            self._segment_data_queue.put(data, timeout=60)
-                        data = self._fetch_segment(seg.absolute_uri)
-                        self._segment_data_queue.put(data, timeout=60)
-                    except requests.exceptions.HTTPError as e:
-                        logger.warning(f'Failed to fetch segment: {repr(e)}')
-                        if e.response.status_code in (403, 404, 599):
-                            break
-                    except requests.exceptions.ConnectionError as e:
-                        logger.warning(repr(e))
-                        self._connection_recovered.wait()
-                    except RetryError as e:
-                        logger.warning(repr(e))
-                        break
-                    else:
-                        break
-
-    def _run_segment_data_feeder(self) -> None:
-        logger.debug('Segment data feeder thread started')
-        try:
-            self._segment_data_feeder()
-        except Exception as e:
-            logger.exception(e)
-            self._handle_exception(e)
-        finally:
-            logger.debug('Segment data feeder thread stopped')
-
-    def _segment_data_feeder(self) -> None:
-        assert self._stream_remuxer is not None
-        bytes_io = io.BytesIO()
-        segment_count = 0
-
-        def on_next(profile: StreamProfile) -> None:
-            self._stream_profile = profile
-
-        def on_error(e: Exception) -> None:
-            logger.warning(f'Failed to analyse stream: {repr(e)}')
-
-        while not self._stopped:
-            try:
-                data = self._segment_data_queue.get(timeout=1)
-            except Empty:
-                continue
-            else:
-                with self._stream_analysed_lock:
-                    if not self._stream_analysed:
-                        if self._init_section_data and not bytes_io.getvalue():
-                            bytes_io.write(self._init_section_data)
-                        else:
-                            bytes_io.write(data)
-                            segment_count += 1
-
-                        if segment_count >= 3:
-                            ffprobe(bytes_io.getvalue()).subscribe(
-                                on_next, on_error
-                            )
-                            bytes_io = io.BytesIO()
-                            segment_count = 0
-                            self._stream_analysed = True
-            try:
-                self._stream_remuxer.input.write(data)
-            except BrokenPipeError:
-                return
-
-    def _run_stream_processor(self) -> None:
-        logger.debug('Stream processor thread started')
-        assert self._stream_remuxer is not None
-
-        with tqdm(
-            desc='Recording',
-            unit='B',
-            unit_scale=True,
-            postfix=self._make_pbar_postfix(),
-        ) as progress_bar:
-            self._progress_bar = progress_bar
-
-            def update_size(size: int) -> None:
-                progress_bar.update(size)
-                self._rec_calculator.submit(size)
-
-            self._stream_processor = StreamProcessor(
-                self._file_manager,
-                filesize_limit=self._filesize_limit,
-                duration_limit=self._duration_limit,
-                analyse_data=True,
-                dedup_join=True,
-                save_extra_metadata=True,
-            )
-            self._stream_processor.size_updates.subscribe(update_size)
-
-            try:
-                self._stream_host_available.wait()
-                self._stream_processor.set_metadata(self._make_metadata())
-                self._stream_processor.process_stream(
-                    StreamProxy(self._stream_remuxer.output),  # type: ignore
-                )
-            except Exception as e:
-                if not self._stopped:
-                    logger.exception(e)
-                    self._handle_exception(e)
-            finally:
-                self._stream_processor.finalize()
-                self._progress_bar = None
-                self._rec_calculator.freeze()
-                logger.debug('Stream processor thread stopped')
-
-    @retry(
-        retry=retry_if_exception_type((
-            requests.exceptions.Timeout,
-            urllib3.exceptions.TimeoutError,
-            urllib3.exceptions.ProtocolError,
-        )),
-        wait=wait_exponential(multiplier=0.1, max=1),
-        stop=stop_after_delay(10),
-    )
-    def _fetch_playlist(self, url: str) -> str:
-        response = self._session.get(url, timeout=3)
-        response.raise_for_status()
-        response.encoding = 'utf-8'
-        return response.text
-
-    @retry(
-        retry=retry_if_exception_type((
-            requests.exceptions.Timeout,
-            urllib3.exceptions.TimeoutError,
-            urllib3.exceptions.ProtocolError,
-        )),
-        wait=wait_exponential(multiplier=0.1, max=5),
-        stop=stop_after_delay(60),
-    )
-    def _fetch_segment(self, url: str) -> bytes:
-        with self._session.get(url, stream=True, timeout=10) as response:
-            response.raise_for_status()
-
-            bytes_io = io.BytesIO()
-            for chunk in response:
-                bytes_io.write(chunk)
-                self._dl_calculator.submit(len(chunk))
-
-            return bytes_io.getvalue()

src/blrec/core/hls_stream_recorder_impl.py

@@ -0,0 +1,91 @@
+import logging
+from typing import Optional
+
+from reactivex import operators as ops
+from reactivex.scheduler import NewThreadScheduler
+
+from ..bili.live import Live
+from ..bili.typing import QualityNumber
+from ..flv import operators as flv_ops
+from . import operators as core_ops
+from .stream_recorder_impl import StreamRecorderImpl
+
+__all__ = ('HLSStreamRecorderImpl',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class HLSStreamRecorderImpl(StreamRecorderImpl):
+    def __init__(
+        self,
+        live: Live,
+        out_dir: str,
+        path_template: str,
+        *,
+        quality_number: QualityNumber = 10000,
+        buffer_size: Optional[int] = None,
+        read_timeout: Optional[int] = None,
+        disconnection_timeout: Optional[int] = None,
+        filesize_limit: int = 0,
+        duration_limit: int = 0,
+    ) -> None:
+        super().__init__(
+            live=live,
+            out_dir=out_dir,
+            path_template=path_template,
+            stream_format='fmp4',
+            quality_number=quality_number,
+            buffer_size=buffer_size,
+            read_timeout=read_timeout,
+            disconnection_timeout=disconnection_timeout,
+            filesize_limit=filesize_limit,
+            duration_limit=duration_limit,
+        )
+
+        self._playlist_fetcher = core_ops.PlaylistFetcher(self._live, self._session)
+        self._playlist_resolver = core_ops.PlaylistResolver()
+        self._segment_fetcher = core_ops.SegmentFetcher(self._live, self._session)
+        self._segment_remuxer = core_ops.SegmentRemuxer(live)
+
+    def _run(self) -> None:
+        self._subscription = (
+            self._stream_param_holder.get_stream_params()  # type: ignore
+            .pipe(
+                self._stream_url_resolver,
+                ops.subscribe_on(
+                    NewThreadScheduler(self._thread_factory('PlaylistFetcher'))
+                ),
+                self._playlist_fetcher,
+                self._request_exception_handler,
+                self._connection_error_handler,
+                self._playlist_resolver,
+                ops.observe_on(
+                    NewThreadScheduler(self._thread_factory('SegmentFetcher'))
+                ),
+                self._segment_fetcher,
+                self._dl_statistics,
+                self._prober,
+                ops.observe_on(
+                    NewThreadScheduler(self._thread_factory('SegmentRemuxer'))
+                ),
+                self._segment_remuxer,
+                ops.observe_on(
+                    NewThreadScheduler(self._thread_factory('StreamRecorder'))
+                ),
+                self._stream_parser,
+                flv_ops.process(),
+                self._cutter,
+                self._limiter,
+                self._join_point_extractor,
+                self._injector,
+                self._analyser,
+                self._dumper,
+                self._rec_statistics,
+                self._progress_bar,
+                self._exception_handler,
+            )
+            .subscribe(
+                on_completed=lambda: self._emit_event('stream_recording_completed')
+            )
+        )

src/blrec/core/metadata_provider.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from collections import OrderedDict
+from datetime import datetime, timedelta, timezone
+from typing import TYPE_CHECKING, Any, Dict
+
+from .. import __github__, __prog__, __version__
+from ..bili.helpers import get_quality_name
+from ..bili.live import Live
+
+if TYPE_CHECKING:
+    from .stream_recorder_impl import StreamRecorderImpl
+
+__all__ = ('MetadataProvider',)
+
+
+class MetadataProvider:
+    def __init__(self, live: Live, stream_recorder: StreamRecorderImpl) -> None:
+        super().__init__()
+        self._live = live
+        self._stream_recorder = stream_recorder
+
+    def __call__(self) -> Dict[str, Any]:
+        return self._make_metadata()
+
+    def _make_metadata(self) -> Dict[str, Any]:
+        live_start_time = datetime.fromtimestamp(
+            self._live.room_info.live_start_time, timezone(timedelta(hours=8))
+        )
+
+        assert self._stream_recorder.real_quality_number is not None
+        stream_quality = '{} ({}{})'.format(
+            get_quality_name(self._stream_recorder.real_quality_number),
+            self._stream_recorder.real_quality_number,
+            ', bluray' if '_bluray' in self._stream_recorder.stream_url else '',
+        )
+
+        return {
+            'Title': self._live.room_info.title,
+            'Artist': self._live.user_info.name,
+            'Date': str(live_start_time),
+            'Comment': f'''\
+B站直播录像
+主播：{self._live.user_info.name}
+标题：{self._live.room_info.title}
+分区：{self._live.room_info.parent_area_name} - {self._live.room_info.area_name}
+房间号：{self._live.room_info.room_id}
+开播时间：{live_start_time}
+流主机: {self._stream_recorder.stream_host}
+流格式：{self._stream_recorder.stream_format}
+流画质：{stream_quality}
+录制程序：{__prog__} v{__version__} {__github__}''',
+            'description': OrderedDict(
+                {
+                    'UserId': str(self._live.user_info.uid),
+                    'UserName': self._live.user_info.name,
+                    'RoomId': str(self._live.room_info.room_id),
+                    'RoomTitle': self._live.room_info.title,
+                    'Area': self._live.room_info.area_name,
+                    'ParentArea': self._live.room_info.parent_area_name,
+                    'LiveStartTime': str(live_start_time),
+                    'StreamHost': self._stream_recorder.stream_host,
+                    'StreamFormat': self._stream_recorder.stream_format,
+                    'StreamQuality': stream_quality,
+                    'Recorder': f'{__prog__} v{__version__} {__github__}',
+                }
+            ),
+        }

src/blrec/core/operators/__init__.py

@@ -0,0 +1,34 @@
+from .connection_error_handler import ConnectionErrorHandler
+from .exception_handler import ExceptionHandler
+from .hls_prober import HLSProber, StreamProfile
+from .playlist_fetcher import PlaylistFetcher
+from .playlist_resolver import PlaylistResolver
+from .progress_bar import ProgressBar
+from .request_exception_handler import RequestExceptionHandler
+from .segment_fetcher import InitSectionData, SegmentData, SegmentFetcher
+from .segment_remuxer import SegmentRemuxer
+from .sized_statistics import SizedStatistics
+from .stream_fetcher import StreamFetcher
+from .stream_parser import StreamParser
+from .stream_statistics import StreamStatistics
+from .stream_url_resolver import StreamURLResolver
+
+__all__ = (
+    'ConnectionErrorHandler',
+    'ExceptionHandler',
+    'HLSProber',
+    'InitSectionData',
+    'PlaylistFetcher',
+    'PlaylistResolver',
+    'ProgressBar',
+    'RequestExceptionHandler',
+    'SegmentData',
+    'SegmentFetcher',
+    'SegmentRemuxer',
+    'SizedStatistics',
+    'StreamFetcher',
+    'StreamParser',
+    'StreamProfile',
+    'StreamStatistics',
+    'StreamURLResolver',
+)

src/blrec/core/operators/connection_error_handler.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import logging
+import time
+from typing import Optional, TypeVar
+
+import aiohttp
+import requests
+from reactivex import Observable, abc
+
+from ...bili.live import Live
+from ...utils import operators as utils_ops
+from ...utils.mixins import AsyncCooperationMixin
+
+__all__ = ('ConnectionErrorHandler',)
+
+
+logger = logging.getLogger(__name__)
+
+_T = TypeVar('_T')
+
+
+class ConnectionErrorHandler(AsyncCooperationMixin):
+    def __init__(
+        self,
+        live: Live,
+        *,
+        disconnection_timeout: Optional[int] = None,
+        check_interval: int = 3,
+    ) -> None:
+        super().__init__()
+        self._live = live
+        self.disconnection_timeout = disconnection_timeout or 600  # seconds
+        self.check_interval = check_interval
+
+    def __call__(self, source: Observable[_T]) -> Observable[_T]:
+        return self._handle(source).pipe(
+            utils_ops.retry(should_retry=self._should_retry)
+        )
+
+    def _handle(self, source: Observable[_T]) -> Observable[_T]:
+        def subscribe(
+            observer: abc.ObserverBase[_T],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_error(exc: Exception) -> None:
+                try:
+                    raise exc
+                except (
+                    aiohttp.ClientConnectionError,
+                    requests.exceptions.ConnectionError,
+                ) as e:
+                    logger.warning(repr(e))
+                    if self._wait_for_connection_error():
+                        observer.on_error(exc)
+                    else:
+                        observer.on_completed()
+                except Exception:
+                    pass
+                observer.on_error(exc)
+
+            return source.subscribe(
+                observer.on_next, on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    def _should_retry(self, exc: Exception) -> bool:
+        if isinstance(
+            exc, (aiohttp.ClientConnectionError, requests.exceptions.ConnectionError)
+        ):
+            return True
+        else:
+            return False
+
+    def _wait_for_connection_error(self) -> bool:
+        timeout = self.disconnection_timeout
+        logger.info(f'Waiting {timeout} seconds for connection recovery... ')
+        timebase = time.monotonic()
+        while not self._run_coroutine(self._live.check_connectivity()):
+            if timeout is not None and time.monotonic() - timebase > timeout:
+                logger.error(f'Connection not recovered in {timeout} seconds')
+                return False
+            time.sleep(self.check_interval)
+        else:
+            logger.info('Connection recovered')
+            return True

src/blrec/core/operators/exception_handler.py

@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import errno
+import logging
+from typing import Optional, TypeVar
+
+from reactivex import Observable, abc
+
+from ...bili.exceptions import LiveRoomEncrypted, LiveRoomHidden, LiveRoomLocked
+from ...utils import operators as utils_ops
+from ...utils.mixins import AsyncCooperationMixin
+
+__all__ = ('ExceptionHandler',)
+
+
+logger = logging.getLogger(__name__)
+
+_T = TypeVar('_T')
+
+
+class ExceptionHandler(AsyncCooperationMixin):
+    def __call__(self, source: Observable[_T]) -> Observable[_T]:
+        return self._handle(source).pipe(utils_ops.retry(delay=1))
+
+    def _handle(self, source: Observable[_T]) -> Observable[_T]:
+        def subscribe(
+            observer: abc.ObserverBase[_T],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_error(exc: Exception) -> None:
+                self._submit_exception(exc)
+                try:
+                    raise exc
+                except OSError as e:
+                    logger.critical(repr(e), exc_info=e)
+                    if e.errno == errno.ENOSPC:
+                        # OSError(28, 'No space left on device')
+                        observer.on_completed()
+                    else:
+                        observer.on_error(exc)
+                except LiveRoomHidden:
+                    logger.error('The live room has been hidden!')
+                    observer.on_completed()
+                except LiveRoomLocked:
+                    logger.error('The live room has been locked!')
+                    observer.on_completed()
+                except LiveRoomEncrypted:
+                    logger.error('The live room has been encrypted!')
+                    observer.on_completed()
+                except Exception:
+                    observer.on_error(exc)
+
+            return source.subscribe(
+                observer.on_next, on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)

src/blrec/core/operators/hls_prober.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import io
+import logging
+from typing import List, Optional, Union
+
+from reactivex import Observable, Subject, abc
+
+from ...utils.ffprobe import StreamProfile, ffprobe
+from .segment_fetcher import InitSectionData, SegmentData
+
+__all__ = ('HLSProber', 'StreamProfile')
+
+
+logger = logging.getLogger(__name__)
+
+
+class HLSProber:
+    def __init__(self) -> None:
+        self._profiles: Subject[StreamProfile] = Subject()
+
+    def _reset(self) -> None:
+        self._gathering: bool = False
+        self._gathered_items: List[Union[InitSectionData, SegmentData]] = []
+
+    @property
+    def profiles(self) -> Observable[StreamProfile]:
+        return self._profiles
+
+    def __call__(
+        self, source: Observable[Union[InitSectionData, SegmentData]]
+    ) -> Observable[Union[InitSectionData, SegmentData]]:
+        return self._probe(source)
+
+    def _probe(
+        self, source: Observable[Union[InitSectionData, SegmentData]]
+    ) -> Observable[Union[InitSectionData, SegmentData]]:
+        def subscribe(
+            observer: abc.ObserverBase[Union[InitSectionData, SegmentData]],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            self._reset()
+
+            def on_next(item: Union[InitSectionData, SegmentData]) -> None:
+                if isinstance(item, InitSectionData):
+                    self._gathered_items.clear()
+                    self._gathering = True
+
+                if self._gathering:
+                    self._gathered_items.append(item)
+                    if len(self._gathered_items) >= 10:
+                        try:
+                            self._do_probe()
+                        except Exception as e:
+                            logger.warning(f'Failed to probe stream: {repr(e)}')
+                        finally:
+                            self._gathered_items.clear()
+                            self._gathering = False
+
+                observer.on_next(item)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    def _do_probe(self) -> None:
+        bytes_io = io.BytesIO()
+        for item in self._gathered_items:
+            bytes_io.write(item.payload)
+
+        def on_next(profile: StreamProfile) -> None:
+            self._profiles.on_next(profile)
+
+        def on_error(e: Exception) -> None:
+            logger.warning(f'Failed to probe stream by ffprobe: {repr(e)}')
+
+        ffprobe(bytes_io.getvalue()).subscribe(on_next, on_error)

src/blrec/core/operators/playlist_fetcher.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import logging
+import time
+from datetime import datetime
+from typing import Optional
+
+import m3u8
+import requests
+import urllib3
+from reactivex import Observable, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+from tenacity import retry, retry_if_exception_type, stop_after_delay, wait_exponential
+
+from ...bili.live import Live
+from ...utils.mixins import SupportDebugMixin
+
+__all__ = ('PlaylistFetcher',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class PlaylistFetcher(SupportDebugMixin):
+    def __init__(self, live: Live, session: requests.Session) -> None:
+        super().__init__()
+        self._init_for_debug(live.room_id)
+        self._live = live
+        self._session = session
+
+    def __call__(self, source: Observable[str]) -> Observable[m3u8.M3U8]:
+        return self._fetch(source)
+
+    def _fetch(self, source: Observable[str]) -> Observable[m3u8.M3U8]:
+        def subscribe(
+            observer: abc.ObserverBase[m3u8.M3U8],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            if self._debug:
+                path = '{}/playlist-{}-{}.m3u8'.format(
+                    self._debug_dir,
+                    self._live.room_id,
+                    datetime.now().strftime('%Y-%m-%d-%H%M%S-%f'),
+                )
+                playlist_debug_file = open(path, 'wt', encoding='utf-8')
+
+            disposed = False
+            subscription = SerialDisposable()
+
+            def on_next(url: str) -> None:
+                logger.info(f'Fetching playlist... {url}')
+                while not disposed:
+                    try:
+                        content = self._fetch_playlist(url)
+                        if self._debug:
+                            playlist_debug_file.write(content + '\n')
+                        playlist = m3u8.loads(content, uri=url)
+                        if playlist.is_variant:
+                            url = self._get_best_quality_url(playlist)
+                            logger.debug('Playlist changed to variant playlist')
+                            on_next(url)
+                    except Exception as e:
+                        logger.warning(f'Failed to fetch playlist: {repr(e)}')
+                        observer.on_error(e)
+                    else:
+                        observer.on_next(playlist)
+                        time.sleep(1)
+
+            def dispose() -> None:
+                nonlocal disposed
+                disposed = True
+                if self._debug:
+                    playlist_debug_file.close()
+
+            subscription.disposable = source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)
+
+    def _get_best_quality_url(self, playlist: m3u8.M3U8) -> str:
+        sorted_playlists = sorted(
+            playlist.playlists, key=lambda p: p.stream_info.bandwidth
+        )
+        return sorted_playlists[-1].absolute_uri
+
+    @retry(
+        reraise=True,
+        retry=retry_if_exception_type(
+            (
+                requests.exceptions.Timeout,
+                urllib3.exceptions.TimeoutError,
+                urllib3.exceptions.ProtocolError,
+            )
+        ),
+        wait=wait_exponential(multiplier=0.1, max=1),
+        stop=stop_after_delay(10),
+    )
+    def _fetch_playlist(self, url: str) -> str:
+        response = self._session.get(url, headers=self._live.headers, timeout=3)
+        response.raise_for_status()
+        response.encoding = 'utf-8'
+        return response.text

src/blrec/core/operators/playlist_resolver.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import logging
+from typing import Final, Optional, Set
+
+import m3u8
+import urllib3
+from ordered_set import OrderedSet
+from reactivex import Observable, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+
+__all__ = ('PlaylistResolver',)
+
+
+logger = logging.getLogger(__name__)
+logging.getLogger(urllib3.__name__).setLevel(logging.WARNING)
+
+
+class PlaylistResolver:
+    _MAX_LAST_SEG_URIS: Final[int] = 30
+
+    def __call__(self, source: Observable[m3u8.M3U8]) -> Observable[m3u8.Segment]:
+        return self._solve(source)
+
+    def _solve(self, source: Observable[m3u8.M3U8]) -> Observable[m3u8.Segment]:
+        def subscribe(
+            observer: abc.ObserverBase[m3u8.Segment],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            disposed = False
+            subscription = SerialDisposable()
+            last_seg_uris: OrderedSet[str] = OrderedSet()
+
+            def on_next(playlist: m3u8.M3U8) -> None:
+                curr_seg_uris: Set[str] = set()
+
+                for seg in playlist.segments:
+                    if disposed:
+                        return
+                    curr_seg_uris.add(seg.uri)
+                    if seg.uri not in last_seg_uris:
+                        observer.on_next(seg)
+                        last_seg_uris.add(seg.uri)
+                        if len(last_seg_uris) > self._MAX_LAST_SEG_URIS:
+                            last_seg_uris.pop(0)
+
+                if last_seg_uris and not curr_seg_uris.intersection(last_seg_uris):
+                    logger.debug(
+                        'Segments broken!\n'
+                        f'Last segments uris: {last_seg_uris}\n'
+                        f'Current segments uris: {curr_seg_uris}'
+                    )
+
+                if playlist.is_endlist:
+                    logger.debug('Playlist ended')
+
+            def dispose() -> None:
+                nonlocal disposed
+                disposed = True
+                last_seg_uris.clear()
+
+            subscription.disposable = source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)

src/blrec/core/operators/progress_bar.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from reactivex import Observable, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+from tqdm import tqdm
+
+from ...bili.live import Live
+from ...flv.operators.typing import FLVStream, FLVStreamItem
+
+__all__ = ('ProgressBar',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class ProgressBar:
+    def __init__(self, live: Live) -> None:
+        self._live = live
+        self._pbar: Optional[tqdm] = None
+
+    def update_bar_info(self) -> None:
+        if self._pbar is not None:
+            self._pbar.set_postfix_str(self._make_pbar_postfix())
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._progress(source)
+
+    def _progress(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            subscription = SerialDisposable()
+
+            self._pbar = tqdm(
+                desc='Recording',
+                unit='B',
+                unit_scale=True,
+                unit_divisor=1024,
+                postfix=self._make_pbar_postfix(),
+            )
+
+            def on_next(item: FLVStreamItem) -> None:
+                if self._pbar is not None:
+                    self._pbar.update(len(item))
+                observer.on_next(item)
+
+            def on_completed() -> None:
+                if self._pbar is not None:
+                    self._pbar.close()
+                    self._pbar = None
+                observer.on_completed()
+
+            def dispose() -> None:
+                if self._pbar is not None:
+                    self._pbar.close()
+                    self._pbar = None
+
+            subscription.disposable = source.subscribe(
+                on_next, observer.on_error, on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)
+
+    def _make_pbar_postfix(self) -> str:
+        return '{room_id} - {user_name}: {room_title}'.format(
+            room_id=self._live.room_info.room_id,
+            user_name=self._live.user_info.name,
+            room_title=self._live.room_info.title,
+        )

src/blrec/core/operators/request_exception_handler.py

@@ -0,0 +1,69 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Optional, TypeVar
+
+import requests
+import urllib3
+from reactivex import Observable, abc
+
+from ...utils import operators as utils_ops
+
+__all__ = ('RequestExceptionHandler',)
+
+
+logger = logging.getLogger(__name__)
+
+_T = TypeVar('_T')
+
+
+class RequestExceptionHandler:
+    def __call__(self, source: Observable[_T]) -> Observable[_T]:
+        return self._handle(source).pipe(
+            utils_ops.retry(delay=1, should_retry=self._should_retry)
+        )
+
+    def _handle(self, source: Observable[_T]) -> Observable[_T]:
+        def subscribe(
+            observer: abc.ObserverBase[_T],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_error(exc: Exception) -> None:
+                try:
+                    raise exc
+                except asyncio.exceptions.TimeoutError:
+                    logger.warning(repr(exc))
+                except requests.exceptions.Timeout:
+                    logger.warning(repr(exc))
+                except requests.exceptions.HTTPError:
+                    logger.warning(repr(exc))
+                except urllib3.exceptions.TimeoutError:
+                    logger.warning(repr(exc))
+                except urllib3.exceptions.ProtocolError:
+                    # ProtocolError('Connection broken: IncompleteRead(
+                    logger.warning(repr(exc))
+                except Exception:
+                    pass
+                observer.on_error(exc)
+
+            return source.subscribe(
+                observer.on_next, on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    def _should_retry(self, exc: Exception) -> bool:
+        if isinstance(
+            exc,
+            (
+                asyncio.exceptions.TimeoutError,
+                requests.exceptions.Timeout,
+                requests.exceptions.HTTPError,
+                urllib3.exceptions.TimeoutError,
+                urllib3.exceptions.ProtocolError,
+            ),
+        ):
+            return True
+        else:
+            return False

src/blrec/core/operators/segment_fetcher.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import logging
+from typing import Optional, Union
+
+import attr
+import m3u8
+import requests
+import urllib3
+from m3u8.model import InitializationSection
+from reactivex import Observable, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+from tenacity import retry, retry_if_exception_type, stop_after_delay, wait_exponential
+
+from ...bili.live import Live
+
+__all__ = ('SegmentFetcher', 'InitSectionData', 'SegmentData')
+
+
+logger = logging.getLogger(__name__)
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True)
+class InitSectionData:
+    payload: bytes
+
+    def __len__(self) -> int:
+        return len(self.payload)
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True)
+class SegmentData:
+    payload: bytes
+
+    def __len__(self) -> int:
+        return len(self.payload)
+
+
+class SegmentFetcher:
+    def __init__(self, live: Live, session: requests.Session) -> None:
+        self._live = live
+        self._session = session
+
+    def __call__(
+        self, source: Observable[m3u8.Segment]
+    ) -> Observable[Union[InitSectionData, SegmentData]]:
+        return self._fetch(source)
+
+    def _fetch(
+        self, source: Observable[m3u8.Segment]
+    ) -> Observable[Union[InitSectionData, SegmentData]]:
+        def subscribe(
+            observer: abc.ObserverBase[Union[InitSectionData, SegmentData]],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            disposed = False
+            subscription = SerialDisposable()
+            init_section: Optional[InitializationSection] = None
+
+            def on_next(seg: m3u8.Segment) -> None:
+                nonlocal init_section
+                url: str = ''
+                try:
+                    if getattr(seg, 'init_section', None) and (
+                        not init_section or seg.init_section.uri != init_section.uri
+                    ):
+                        url = seg.init_section.absolute_uri
+                        data = self._fetch_segment(url)
+                        init_section = seg.init_section
+                        observer.on_next(InitSectionData(payload=data))
+                    url = seg.absolute_uri
+                    data = self._fetch_segment(url)
+                    observer.on_next(SegmentData(payload=data))
+                except Exception as e:
+                    logger.warning(f'Failed to fetch segment {url}: {repr(e)}')
+
+            def dispose() -> None:
+                nonlocal disposed
+                disposed = True
+
+            subscription.disposable = source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)
+
+    @retry(
+        reraise=True,
+        retry=retry_if_exception_type(
+            (
+                requests.exceptions.Timeout,
+                requests.exceptions.HTTPError,
+                urllib3.exceptions.TimeoutError,
+                urllib3.exceptions.ProtocolError,
+            )
+        ),
+        wait=wait_exponential(multiplier=0.1, max=5),
+        stop=stop_after_delay(60),
+    )
+    def _fetch_segment(self, url: str) -> bytes:
+        with self._session.get(url, headers=self._live.headers, timeout=10) as response:
+            response.raise_for_status()
+            return response.content

src/blrec/core/operators/segment_remuxer.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import io
+import logging
+from typing import Final, List, Optional, Union
+
+import urllib3
+from reactivex import Observable, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+
+from ...bili.live import Live
+from ...utils.io import wait_for
+from ..stream_remuxer import StreamRemuxer
+from .segment_fetcher import InitSectionData, SegmentData
+
+__all__ = ('SegmentRemuxer',)
+
+
+logger = logging.getLogger(__name__)
+logging.getLogger(urllib3.__name__).setLevel(logging.WARNING)
+
+
+class SegmentRemuxer:
+    _MAX_SEGMENT_DATA_CACHE: Final = 3
+
+    def __init__(self, live: Live) -> None:
+        self._live = live
+        self._stream_remuxer = StreamRemuxer(live.room_id, remove_filler_data=True)
+
+    def __call__(
+        self, source: Observable[Union[InitSectionData, SegmentData]]
+    ) -> Observable[io.RawIOBase]:
+        return self._remux(source)
+
+    def _remux(
+        self, source: Observable[Union[InitSectionData, SegmentData]]
+    ) -> Observable[io.RawIOBase]:
+        def subscribe(
+            observer: abc.ObserverBase[io.RawIOBase],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            disposed = False
+            subscription = SerialDisposable()
+
+            init_section_data: Optional[bytes] = None
+            segment_data_cache: List[bytes] = []
+
+            self._stream_remuxer.stop()
+
+            def on_next(data: Union[InitSectionData, SegmentData]) -> None:
+                nonlocal init_section_data
+
+                if isinstance(data, InitSectionData):
+                    init_section_data = data.payload
+
+                try:
+                    if self._stream_remuxer.stopped:
+                        self._stream_remuxer.start()
+                        while True:
+                            ready = self._stream_remuxer.wait(timeout=1)
+                            if disposed:
+                                return
+                            if ready:
+                                break
+
+                        observer.on_next(RemuxedStream(self._stream_remuxer))
+
+                        if segment_data_cache:
+                            if init_section_data:
+                                self._stream_remuxer.input.write(init_section_data)
+                            for cached_data in segment_data_cache:
+                                if cached_data == init_section_data:
+                                    continue
+                                self._stream_remuxer.input.write(cached_data)
+
+                    self._stream_remuxer.input.write(data.payload)
+                except Exception as exc:
+                    logger.warning(f'Failed to write data to stream remuxer: {exc}')
+                    self._stream_remuxer.stop()
+
+                segment_data_cache.append(data.payload)
+                if len(segment_data_cache) > self._MAX_SEGMENT_DATA_CACHE:
+                    segment_data_cache.pop(0)
+
+            def dispose() -> None:
+                nonlocal disposed
+                disposed = True
+
+            subscription.disposable = source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)
+
+
+class RemuxedStream(io.RawIOBase):
+    def __init__(
+        self, stream_remuxer: StreamRemuxer, *, read_timeout: float = 10
+    ) -> None:
+        self._stream_remuxer = stream_remuxer
+        self._read_timmeout = read_timeout
+        self._offset: int = 0
+
+    def read(self, size: int = -1) -> bytes:
+        try:
+            data = wait_for(
+                self._stream_remuxer.output.read,
+                args=(size,),
+                timeout=self._read_timmeout,
+            )
+        except Exception as exc:
+            logger.warning(f'Failed to read data from stream remuxer: {exc}')
+            self._stream_remuxer.stop()
+            raise EOFError
+        else:
+            assert data is not None
+            self._offset += len(data)
+            return data
+
+    def tell(self) -> int:
+        return self._offset

src/blrec/core/operators/sized_statistics.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from typing import Optional, Sized
+
+from reactivex import Observable, abc
+
+from ..statistics import Statistics
+
+__all__ = ('SizedStatistics',)
+
+
+class SizedStatistics:
+    def __init__(self) -> None:
+        self._statistics = Statistics()
+
+    @property
+    def count(self) -> int:
+        return self._statistics.count
+
+    @property
+    def rate(self) -> float:
+        return self._statistics.rate
+
+    @property
+    def elapsed(self) -> float:
+        return self._statistics.elapsed
+
+    def freeze(self) -> None:
+        self._statistics.freeze()
+
+    def reset(self) -> None:
+        self._statistics.reset()
+
+    def __call__(self, source: Observable[Sized]) -> Observable[Sized]:
+        self.reset()
+        return self._calc(source)
+
+    def _calc(self, source: Observable[Sized]) -> Observable[Sized]:
+        def subscribe(
+            observer: abc.ObserverBase[Sized],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_next(item: Sized) -> None:
+                self._statistics.submit(len(item))
+                observer.on_next(item)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)

src/blrec/core/operators/stream_fetcher.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import io
+import logging
+from typing import Optional
+
+import requests
+from reactivex import Observable, abc
+
+from ...bili.live import Live
+from ...utils.mixins import AsyncCooperationMixin
+
+__all__ = ('StreamFetcher',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class StreamFetcher(AsyncCooperationMixin):
+    def __init__(
+        self,
+        live: Live,
+        session: requests.Session,
+        *,
+        read_timeout: Optional[int] = None,
+    ) -> None:
+        super().__init__()
+        self._live = live
+        self._session = session
+        self.read_timeout = read_timeout or 3
+
+    def __call__(self, source: Observable[str]) -> Observable[io.RawIOBase]:
+        return self._fetch(source)
+
+    def _fetch(self, source: Observable[str]) -> Observable[io.RawIOBase]:
+        def subscribe(
+            observer: abc.ObserverBase[io.RawIOBase],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_next(url: str) -> None:
+                try:
+                    logger.info(f'Requesting live stream... {url}')
+                    response = self._session.get(
+                        url,
+                        stream=True,
+                        headers=self._live.headers,
+                        timeout=self.read_timeout,
+                    )
+                    logger.info('Response received')
+                    response.raise_for_status()
+                except Exception as e:
+                    logger.warning(f'Failed to request live stream: {repr(e)}')
+                    observer.on_error(e)
+                else:
+                    observer.on_next(response.raw)  # urllib3.response.HTTPResponse
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)

src/blrec/core/operators/stream_parser.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import io
+import logging
+
+from reactivex import Observable
+from reactivex import operators as ops
+
+from ...flv import operators as flv_ops
+from ...flv.exceptions import FlvDataError
+from ...flv.operators.typing import FLVStream
+from ...utils import operators as utils_ops
+from ..stream_param_holder import StreamParamHolder
+
+__all__ = ('StreamParser',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class StreamParser:
+    def __init__(
+        self, stream_param_holder: StreamParamHolder, *, ignore_eof: bool = False
+    ) -> None:
+        self._stream_param_holder = stream_param_holder
+        self._ignore_eof = ignore_eof
+
+    def __call__(self, source: Observable[io.RawIOBase]) -> FLVStream:
+        return source.pipe(  # type: ignore
+            flv_ops.parse(ignore_eof=self._ignore_eof, backup_timestamp=True),
+            ops.do_action(on_error=self._before_retry),
+            utils_ops.retry(should_retry=self._should_retry),
+        )
+
+    def _should_retry(self, exc: Exception) -> bool:
+        if isinstance(exc, (EOFError, FlvDataError)):
+            return True
+        else:
+            return False
+
+    def _before_retry(self, exc: Exception) -> None:
+        try:
+            raise exc
+        except EOFError:
+            logger.debug(repr(exc))
+        except FlvDataError:
+            logger.warning(f'Failed to parse stream: {repr(exc)}')
+            if not self._stream_param_holder.use_alternative_stream:
+                self._stream_param_holder.use_alternative_stream = True
+            else:
+                self._stream_param_holder.use_alternative_stream = False
+                self._stream_param_holder.rotate_api_platform()
+        except Exception:
+            pass

src/blrec/core/operators/stream_statistics.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import io
+from typing import Any, Optional
+
+from reactivex import Observable, Subject, abc
+
+from ..statistics import Statistics
+
+__all__ = ('StreamStatistics',)
+
+
+class StreamStatistics:
+    def __init__(self) -> None:
+        self._statistics = Statistics()
+
+    @property
+    def count(self) -> int:
+        return self._statistics.count
+
+    @property
+    def rate(self) -> float:
+        return self._statistics.rate
+
+    @property
+    def elapsed(self) -> float:
+        return self._statistics.elapsed
+
+    def freeze(self) -> None:
+        self._statistics.freeze()
+
+    def reset(self) -> None:
+        self._statistics.reset()
+
+    def __call__(self, source: Observable[io.RawIOBase]) -> Observable[io.RawIOBase]:
+        self.reset()
+        return self._calc(source)
+
+    def _calc(self, source: Observable[io.RawIOBase]) -> Observable[io.RawIOBase]:
+        def subscribe(
+            observer: abc.ObserverBase[io.RawIOBase],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_next(stream: io.RawIOBase) -> None:
+                calculable_stream = CalculableStream(stream)
+                calculable_stream.size_updates.subscribe(self._statistics.submit)
+                observer.on_next(calculable_stream)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+
+class CalculableStream(io.RawIOBase):
+    def __init__(self, stream: io.RawIOBase) -> None:
+        self._stream = stream
+        self._offset: int = 0
+        self._size_updates: Subject[int] = Subject()
+
+    @property
+    def size_updates(self) -> Observable[int]:
+        return self._size_updates
+
+    def read(self, size: int = -1) -> bytes:
+        data = self._stream.read(size)
+        assert data is not None
+        self._offset += len(data)
+        self._size_updates.on_next(len(data))
+        return data
+
+    def readinto(self, b: Any) -> int:
+        n = self._stream.readinto(b)
+        assert n is not None
+        self._offset += n
+        self._size_updates.on_next(n)
+        return n
+
+    def tell(self) -> int:
+        return self._offset

src/blrec/core/operators/stream_url_resolver.py

@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+import logging
+from typing import Optional
+from urllib.parse import urlparse
+
+import requests
+import urllib3
+from reactivex import Observable, abc
+from reactivex import operators as ops
+
+from ...bili.exceptions import (
+    LiveRoomEncrypted,
+    LiveRoomHidden,
+    LiveRoomLocked,
+    NoAlternativeStreamAvailable,
+    NoStreamAvailable,
+    NoStreamCodecAvailable,
+    NoStreamFormatAvailable,
+    NoStreamQualityAvailable,
+)
+from ...bili.live import Live
+from ...utils import operators as utils_ops
+from ...utils.mixins import AsyncCooperationMixin
+from ..stream_param_holder import StreamParamHolder, StreamParams
+
+__all__ = ('StreamURLResolver',)
+
+
+logger = logging.getLogger(__name__)
+logging.getLogger(urllib3.__name__).setLevel(logging.WARNING)
+
+
+class StreamURLResolver(AsyncCooperationMixin):
+    def __init__(self, live: Live, stream_param_holder: StreamParamHolder) -> None:
+        super().__init__()
+        self._live = live
+        self._stream_param_holder = stream_param_holder
+        self._stream_url: str = ''
+        self._stream_host: str = ''
+        self._stream_params: Optional[StreamParams] = None
+
+    @property
+    def stream_url(self) -> str:
+        return self._stream_url
+
+    @property
+    def stream_host(self) -> str:
+        return self._stream_host
+
+    def _reset(self) -> None:
+        self._stream_url = ''
+        self._stream_host = ''
+        self._stream_params = None
+
+    def __call__(self, source: Observable[StreamParams]) -> Observable[str]:
+        self._reset()
+        return self._solve(source).pipe(
+            ops.do_action(on_error=self._before_retry),
+            utils_ops.retry(delay=1, should_retry=self._should_retry),
+        )
+
+    def _solve(self, source: Observable[StreamParams]) -> Observable[str]:
+        def subscribe(
+            observer: abc.ObserverBase[str],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_next(params: StreamParams) -> None:
+                if self._can_resue_url(params):
+                    observer.on_next(self._stream_url)
+                    return
+
+                try:
+                    logger.info(
+                        f'Getting the live stream url... '
+                        f'qn: {params.quality_number}, '
+                        f'format: {params.stream_format}, '
+                        f'api platform: {params.api_platform}, '
+                        f'use alternative stream: {params.use_alternative_stream}'
+                    )
+                    url = self._run_coroutine(
+                        self._live.get_live_stream_url(
+                            params.quality_number,
+                            api_platform=params.api_platform,
+                            stream_format=params.stream_format,
+                            select_alternative=params.use_alternative_stream,
+                        )
+                    )
+                except Exception as e:
+                    logger.warning(f'Failed to get live stream url: {repr(e)}')
+                    observer.on_error(e)
+                else:
+                    logger.info(f"Got live stream url: '{url}'")
+                    self._stream_url = url
+                    self._stream_host = urlparse(url).hostname or ''
+                    self._stream_params = params
+                    observer.on_next(url)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    def _can_resue_url(self, params: StreamParams) -> bool:
+        if params == self._stream_params and self._stream_url:
+            try:
+                response = requests.get(
+                    self._stream_url, stream=True, headers=self._live.headers
+                )
+                response.raise_for_status()
+            except Exception:
+                return False
+            else:
+                return True
+        else:
+            return False
+
+    def _should_retry(self, exc: Exception) -> bool:
+        if isinstance(
+            exc,
+            (
+                NoStreamAvailable,
+                NoStreamCodecAvailable,
+                NoStreamFormatAvailable,
+                NoStreamQualityAvailable,
+                NoAlternativeStreamAvailable,
+            ),
+        ):
+            return True
+        else:
+            return False
+
+    def _before_retry(self, exc: Exception) -> None:
+        try:
+            raise exc
+        except (NoStreamAvailable, NoStreamCodecAvailable, NoStreamFormatAvailable):
+            pass
+        except NoStreamQualityAvailable:
+            qn = self._stream_param_holder.quality_number
+            logger.info(
+                f'The specified stream quality ({qn}) is not available, '
+                'will using the original stream quality (10000) instead.'
+            )
+            self._stream_param_holder.fall_back_quality()
+        except NoAlternativeStreamAvailable:
+            logger.debug(
+                'No alternative stream url available, '
+                'will using the primary stream url instead.'
+            )
+            self._stream_param_holder.use_alternative_stream = False
+            self._stream_param_holder.rotate_api_platform()
+        except LiveRoomHidden:
+            pass
+        except LiveRoomLocked:
+            pass
+        except LiveRoomEncrypted:
+            pass
+        except Exception:
+            pass

src/blrec/core/path_provider.py

@@ -0,0 +1,81 @@
+import asyncio
+import logging
+import os
+import re
+import time
+from datetime import datetime
+from typing import Tuple
+
+import aiohttp
+from tenacity import retry, retry_if_exception_type, stop_after_delay, wait_exponential
+
+from ..bili.live import Live
+from ..path import escape_path
+from ..utils.mixins import AsyncCooperationMixin
+
+__all__ = ('PathProvider',)
+
+logger = logging.getLogger(__name__)
+
+
+class PathProvider(AsyncCooperationMixin):
+    def __init__(self, live: Live, out_dir: str, path_template: str) -> None:
+        super().__init__()
+        self._live = live
+        self.out_dir = out_dir
+        self.path_template = path_template
+
+    def __call__(self) -> Tuple[str, int]:
+        timestamp = self._get_timestamp()
+        path = self._make_path(timestamp)
+        return path, timestamp
+
+    def _get_timestamp(self) -> int:
+        try:
+            return self._get_server_timestamp()
+        except Exception as e:
+            logger.warning(f'Failed to get server timestamp: {repr(e)}')
+            return self._get_local_timestamp()
+
+    def _get_local_timestamp(self) -> int:
+        return int(time.time())
+
+    @retry(
+        reraise=True,
+        retry=retry_if_exception_type((asyncio.TimeoutError, aiohttp.ClientError)),
+        wait=wait_exponential(multiplier=0.1, max=1),
+        stop=stop_after_delay(3),
+    )
+    def _get_server_timestamp(self) -> int:
+        return self._run_coroutine(self._live.get_server_timestamp())
+
+    def _make_path(self, timestamp: int) -> str:
+        date_time = datetime.fromtimestamp(timestamp)
+        relpath = self.path_template.format(
+            roomid=self._live.room_id,
+            uname=escape_path(self._live.user_info.name),
+            title=escape_path(self._live.room_info.title),
+            area=escape_path(self._live.room_info.area_name),
+            parent_area=escape_path(self._live.room_info.parent_area_name),
+            year=date_time.year,
+            month=str(date_time.month).rjust(2, '0'),
+            day=str(date_time.day).rjust(2, '0'),
+            hour=str(date_time.hour).rjust(2, '0'),
+            minute=str(date_time.minute).rjust(2, '0'),
+            second=str(date_time.second).rjust(2, '0'),
+        )
+
+        pathname = os.path.abspath(
+            os.path.expanduser(os.path.join(self.out_dir, relpath) + '.flv')
+        )
+        os.makedirs(os.path.dirname(pathname), exist_ok=True)
+        while os.path.exists(pathname):
+            root, ext = os.path.splitext(pathname)
+            m = re.search(r'_\((\d+)\)$', root)
+            if m is None:
+                root += '_(1)'
+            else:
+                root = re.sub(r'\(\d+\)$', f'({int(m.group(1)) + 1})', root)
+            pathname = root + ext
+
+        return pathname

src/blrec/core/raw_danmaku_dumper.py

@@ -12,9 +12,7 @@ from tenacity import (
 )
 
 from .raw_danmaku_receiver import RawDanmakuReceiver
-from .base_stream_recorder import (
-    BaseStreamRecorder, StreamRecorderEventListener
-)
+from .stream_recorder import StreamRecorder, StreamRecorderEventListener
 from ..bili.live import Live
 from ..exception import exception_callback, submit_exception
 from ..event.event_emitter import EventListener, EventEmitter
@@ -45,7 +43,7 @@ class RawDanmakuDumper(
     def __init__(
         self,
         live: Live,
-        stream_recorder: BaseStreamRecorder,
+        stream_recorder: StreamRecorder,
         danmaku_receiver: RawDanmakuReceiver,
     ) -> None:
         super().__init__()
@@ -53,13 +51,7 @@ class RawDanmakuDumper(
         self._stream_recorder = stream_recorder
         self._receiver = danmaku_receiver
 
-    def change_stream_recorder(
-        self, stream_recorder: BaseStreamRecorder
-    ) -> None:
-        self._stream_recorder.remove_listener(self)
-        self._stream_recorder = stream_recorder
-        self._stream_recorder.add_listener(self)
-        logger.debug('Changed stream recorder')
+        self._lock: asyncio.Lock = asyncio.Lock()
 
     def _do_enable(self) -> None:
         self._stream_recorder.add_listener(self)
@@ -72,11 +64,13 @@ class RawDanmakuDumper(
     async def on_video_file_created(
         self, video_path: str, record_start_time: int
     ) -> None:
-        self._path = raw_danmaku_path(video_path)
-        self._start_dumping()
+        async with self._lock:
+            self._path = raw_danmaku_path(video_path)
+            self._start_dumping()
 
     async def on_video_file_completed(self, video_path: str) -> None:
-        await self._stop_dumping()
+        async with self._lock:
+            await self._stop_dumping()
 
     def _start_dumping(self) -> None:
         self._create_dump_task()

src/blrec/core/recorder.py

@@ -1,35 +1,27 @@
 from __future__ import annotations
+
 import asyncio
 import logging
 from datetime import datetime
-from typing import Iterator, Optional, Type
+from typing import Iterator, Optional
 
-import aiohttp
-import aiofiles
 import humanize
-from tenacity import retry, wait_fixed, stop_after_attempt
 
-from .danmaku_receiver import DanmakuReceiver
-from .danmaku_dumper import DanmakuDumper, DanmakuDumperEventListener
-from .raw_danmaku_receiver import RawDanmakuReceiver
-from .raw_danmaku_dumper import RawDanmakuDumper, RawDanmakuDumperEventListener
-from .base_stream_recorder import (
-    BaseStreamRecorder, StreamRecorderEventListener
-)
-from .stream_analyzer import StreamProfile
-from .flv_stream_recorder import FLVStreamRecorder
-from .hls_stream_recorder import HLSStreamRecorder
-from ..event.event_emitter import EventListener, EventEmitter
-from ..flv.data_analyser import MetaData
-from ..bili.live import Live
-from ..bili.models import RoomInfo
 from ..bili.danmaku_client import DanmakuClient
-from ..bili.live_monitor import LiveMonitor, LiveEventListener
-from ..bili.typing import StreamFormat, QualityNumber
-from ..utils.mixins import AsyncStoppableMixin
-from ..path import cover_path
+from ..bili.live import Live
+from ..bili.live_monitor import LiveEventListener, LiveMonitor
+from ..bili.models import RoomInfo
+from ..bili.typing import QualityNumber, StreamFormat
+from ..event.event_emitter import EventEmitter, EventListener
+from ..flv.operators import MetaData, StreamProfile
 from ..logging.room_id import aio_task_with_room_id
-
+from ..utils.mixins import AsyncStoppableMixin
+from .cover_downloader import CoverDownloader, CoverSaveStrategy
+from .danmaku_dumper import DanmakuDumper, DanmakuDumperEventListener
+from .danmaku_receiver import DanmakuReceiver
+from .raw_danmaku_dumper import RawDanmakuDumper, RawDanmakuDumperEventListener
+from .raw_danmaku_receiver import RawDanmakuReceiver
+from .stream_recorder import StreamRecorder, StreamRecorderEventListener
 
 __all__ = 'RecorderEventListener', 'Recorder'
 
@@ -47,29 +39,19 @@ class RecorderEventListener(EventListener):
     async def on_recording_cancelled(self, recorder: Recorder) -> None:
         ...
 
-    async def on_video_file_created(
-        self, recorder: Recorder, path: str
-    ) -> None:
+    async def on_video_file_created(self, recorder: Recorder, path: str) -> None:
         ...
 
-    async def on_video_file_completed(
-        self, recorder: Recorder, path: str
-    ) -> None:
+    async def on_video_file_completed(self, recorder: Recorder, path: str) -> None:
         ...
 
-    async def on_danmaku_file_created(
-        self, recorder: Recorder, path: str
-    ) -> None:
+    async def on_danmaku_file_created(self, recorder: Recorder, path: str) -> None:
         ...
 
-    async def on_danmaku_file_completed(
-        self, recorder: Recorder, path: str
-    ) -> None:
+    async def on_danmaku_file_completed(self, recorder: Recorder, path: str) -> None:
         ...
 
-    async def on_raw_danmaku_file_created(
-        self, recorder: Recorder, path: str
-    ) -> None:
+    async def on_raw_danmaku_file_created(self, recorder: Recorder, path: str) -> None:
         ...
 
     async def on_raw_danmaku_file_completed(
@@ -96,6 +78,7 @@ class Recorder(
         *,
         stream_format: StreamFormat = 'flv',
         quality_number: QualityNumber = 10000,
+        fmp4_stream_timeout: int = 10,
         buffer_size: Optional[int] = None,
         read_timeout: Optional[int] = None,
         disconnection_timeout: Optional[int] = None,
@@ -107,6 +90,7 @@ class Recorder(
         record_guard_buy: bool = False,
         record_super_chat: bool = False,
         save_cover: bool = False,
+        cover_save_strategy: CoverSaveStrategy = CoverSaveStrategy.DEFAULT,
         save_raw_danmaku: bool = False,
     ) -> None:
         super().__init__()
@@ -114,23 +98,18 @@ class Recorder(
         self._live = live
         self._danmaku_client = danmaku_client
         self._live_monitor = live_monitor
-        self.save_cover = save_cover
         self.save_raw_danmaku = save_raw_danmaku
 
         self._recording: bool = False
         self._stream_available: bool = False
 
-        cls: Type[BaseStreamRecorder]
-        if stream_format == 'flv':
-            cls = FLVStreamRecorder
-        else:
-            cls = HLSStreamRecorder
-        self._stream_recorder = cls(
+        self._stream_recorder = StreamRecorder(
             self._live,
             out_dir=out_dir,
             path_template=path_template,
             stream_format=stream_format,
             quality_number=quality_number,
+            fmp4_stream_timeout=fmp4_stream_timeout,
             buffer_size=buffer_size,
             read_timeout=read_timeout,
             disconnection_timeout=disconnection_timeout,
@@ -151,9 +130,14 @@ class Recorder(
         )
         self._raw_danmaku_receiver = RawDanmakuReceiver(danmaku_client)
         self._raw_danmaku_dumper = RawDanmakuDumper(
+            self._live, self._stream_recorder, self._raw_danmaku_receiver
+        )
+
+        self._cover_downloader = CoverDownloader(
             self._live,
             self._stream_recorder,
-            self._raw_danmaku_receiver,
+            save_cover=save_cover,
+            cover_save_strategy=cover_save_strategy,
         )
 
     @property
@@ -181,11 +165,19 @@ class Recorder(
         self._stream_recorder.quality_number = value
 
     @property
-    def real_stream_format(self) -> StreamFormat:
+    def fmp4_stream_timeout(self) -> int:
+        return self._stream_recorder.fmp4_stream_timeout
+
+    @fmp4_stream_timeout.setter
+    def fmp4_stream_timeout(self, value: int) -> None:
+        self._stream_recorder.fmp4_stream_timeout = value
+
+    @property
+    def real_stream_format(self) -> Optional[StreamFormat]:
         return self._stream_recorder.real_stream_format
 
     @property
-    def real_quality_number(self) -> QualityNumber:
+    def real_quality_number(self) -> Optional[QualityNumber]:
         return self._stream_recorder.real_quality_number
 
     @property
@@ -252,6 +244,22 @@ class Recorder(
     def record_super_chat(self, value: bool) -> None:
         self._danmaku_dumper.record_super_chat = value
 
+    @property
+    def save_cover(self) -> bool:
+        return self._cover_downloader.save_cover
+
+    @save_cover.setter
+    def save_cover(self, value: bool) -> None:
+        self._cover_downloader.save_cover = value
+
+    @property
+    def cover_save_strategy(self) -> CoverSaveStrategy:
+        return self._cover_downloader.cover_save_strategy
+
+    @cover_save_strategy.setter
+    def cover_save_strategy(self, value: CoverSaveStrategy) -> None:
+        self._cover_downloader.cover_save_strategy = value
+
     @property
     def stream_url(self) -> str:
         return self._stream_recorder.stream_url
@@ -375,15 +383,11 @@ class Recorder(
         self._print_changed_room_info(room_info)
         self._stream_recorder.update_progress_bar_info()
 
-    async def on_video_file_created(
-        self, path: str, record_start_time: int
-    ) -> None:
+    async def on_video_file_created(self, path: str, record_start_time: int) -> None:
         await self._emit('video_file_created', self, path)
 
     async def on_video_file_completed(self, path: str) -> None:
         await self._emit('video_file_completed', self, path)
-        if self.save_cover:
-            await self._save_cover_image(path)
 
     async def on_danmaku_file_created(self, path: str) -> None:
         await self._emit('danmaku_file_created', self, path)
@@ -397,8 +401,8 @@ class Recorder(
     async def on_raw_danmaku_file_completed(self, path: str) -> None:
         await self._emit('raw_danmaku_file_completed', self, path)
 
-    async def on_stream_recording_stopped(self) -> None:
-        logger.debug('Stream recording stopped')
+    async def on_stream_recording_completed(self) -> None:
+        logger.debug('Stream recording completed')
         await self._stop_recording()
 
     async def _do_start(self) -> None:
@@ -426,7 +430,6 @@ class Recorder(
     async def _start_recording(self) -> None:
         if self._recording:
             return
-        self._change_stream_recorder()
         self._recording = True
 
         if self.save_raw_danmaku:
@@ -434,6 +437,7 @@ class Recorder(
             self._raw_danmaku_receiver.start()
         self._danmaku_dumper.enable()
         self._danmaku_receiver.start()
+        self._cover_downloader.enable()
 
         await self._prepare()
         if self._stream_available:
@@ -455,6 +459,7 @@ class Recorder(
             self._raw_danmaku_receiver.stop()
         self._danmaku_dumper.disable()
         self._danmaku_receiver.stop()
+        self._cover_downloader.disable()
 
         if self._stopped:
             logger.info('Recording Cancelled')
@@ -492,60 +497,6 @@ class Recorder(
             if not self._stream_recorder.stopped:
                 await self.stop()
 
-    def _change_stream_recorder(self) -> None:
-        if self._recording:
-            logger.debug('Can not change stream recorder while recording')
-            return
-
-        cls: Type[BaseStreamRecorder]
-        if self.stream_format == 'flv':
-            cls = FLVStreamRecorder
-        else:
-            cls = HLSStreamRecorder
-
-        if self._stream_recorder.__class__ == cls:
-            return
-
-        self._stream_recorder.remove_listener(self)
-        self._stream_recorder = cls(
-            self._live,
-            out_dir=self.out_dir,
-            path_template=self.path_template,
-            stream_format=self.stream_format,
-            quality_number=self.quality_number,
-            buffer_size=self.buffer_size,
-            read_timeout=self.read_timeout,
-            disconnection_timeout=self.disconnection_timeout,
-            filesize_limit=self.filesize_limit,
-            duration_limit=self.duration_limit,
-        )
-        self._stream_recorder.add_listener(self)
-
-        self._danmaku_dumper.change_stream_recorder(self._stream_recorder)
-        self._raw_danmaku_dumper.change_stream_recorder(self._stream_recorder)
-
-        logger.debug(f'Changed stream recorder to {cls.__name__}')
-
-    @aio_task_with_room_id
-    async def _save_cover_image(self, video_path: str) -> None:
-        try:
-            await self._live.update_info()
-            url = self._live.room_info.cover
-            ext = url.rsplit('.', 1)[-1]
-            path = cover_path(video_path, ext)
-            await self._save_file(url, path)
-        except Exception as e:
-            logger.error(f'Failed to save cover image: {repr(e)}')
-        else:
-            logger.info(f'Saved cover image: {path}')
-
-    @retry(reraise=True, wait=wait_fixed(1), stop=stop_after_attempt(3))
-    async def _save_file(self, url: str, path: str) -> None:
-        async with aiohttp.ClientSession(raise_for_status=True) as session:
-            async with session.get(url) as response:
-                async with aiofiles.open(path, 'wb') as file:
-                    await file.write(await response.read())
-
     def _print_waiting_message(self) -> None:
         logger.info('Waiting... until the live starts')
 
@@ -554,9 +505,7 @@ class Recorder(
         user_info = self._live.user_info
 
         if room_info.live_start_time > 0:
-            live_start_time = str(
-                datetime.fromtimestamp(room_info.live_start_time)
-            )
+            live_start_time = str(datetime.fromtimestamp(room_info.live_start_time))
         else:
             live_start_time = 'NULL'
 

src/blrec/core/retry.py

@@ -1,60 +0,0 @@
-import logging
-from typing import Any, Callable, Optional, Type, cast
-
-from tenacity import wait_exponential, RetryCallState
-from tenacity import _utils
-
-
-class wait_exponential_for_same_exceptions(wait_exponential):
-    """Wait strategy that applies exponential backoff only for same
-    continuing exceptions.
-    """
-
-    def __init__(
-        self,
-        multiplier: float = 1,
-        max: float = _utils.MAX_WAIT,
-        exp_base: int = 2,
-        min: float = 0,
-        continuing_criteria: float = 5.0,
-    ) -> None:
-        super().__init__(multiplier, max, exp_base, min)
-        self._continuing_criteria = continuing_criteria
-        self._prev_exc_type: Optional[Type[BaseException]] = None
-        self._prev_exc_ts: Optional[float] = None
-        self._last_wait_time: float = 0
-
-    def __call__(self, retry_state: RetryCallState) -> float:
-        if (
-            retry_state.outcome is not None and
-            (exc := retry_state.outcome.exception())
-        ):
-            curr_exc_type = type(exc)
-            curr_exc_ts = cast(float, retry_state.outcome_timestamp)
-            if (
-                curr_exc_type is not self._prev_exc_type or
-                not self._is_continuing(curr_exc_ts)
-            ):
-                retry_state.attempt_number = 1
-                self._prev_exc_type = curr_exc_type
-                self._prev_exc_ts = curr_exc_ts
-
-        self._last_wait_time = wait_time = super().__call__(retry_state)
-        return wait_time
-
-    def _is_continuing(self, curr_exc_ts: float) -> bool:
-        assert self._prev_exc_ts is not None
-        return (
-            curr_exc_ts - (self._prev_exc_ts + self._last_wait_time) <
-            self._continuing_criteria
-        )
-
-
-def before_sleep_log(
-    logger: logging.Logger, log_level: int, name: str = ''
-) -> Callable[[RetryCallState], Any]:
-    def log_it(retry_state: RetryCallState) -> None:
-        seconds = cast(float, getattr(retry_state.next_action, 'sleep'))
-        logger.log(log_level, 'Retry %s after %s seconds', name, seconds)
-
-    return log_it

src/blrec/core/statistics.py

@@ -1,10 +1,10 @@
 import time
 
 
-__all__ = 'StatisticsCalculator',
+__all__ = 'Statistics',
 
 
-class StatisticsCalculator:
+class Statistics:
     def __init__(self, interval: float = 1.0) -> None:
         self._interval = interval
         self._frozen = True

src/blrec/core/stream_param_holder.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Final, Optional
+
+import attr
+from reactivex import Observable, abc, create
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+from reactivex.scheduler.currentthreadscheduler import CurrentThreadScheduler
+
+from ..bili.typing import ApiPlatform, QualityNumber, StreamFormat
+
+__all__ = ('StreamParamHolder',)
+
+
+logger = logging.getLogger(__name__)
+
+
+@attr.s(auto_attribs=True, frozen=True, slots=True)
+class StreamParams:
+    stream_format: StreamFormat
+    quality_number: QualityNumber
+    api_platform: ApiPlatform
+    use_alternative_stream: bool
+
+
+class StreamParamHolder:
+    def __init__(
+        self,
+        *,
+        stream_format: StreamFormat = 'flv',
+        quality_number: QualityNumber = 10000,
+        api_platform: ApiPlatform = 'web',
+        use_alternative_stream: bool = False,
+    ) -> None:
+        super().__init__()
+        self._stream_format: Final = stream_format
+        self._quality_number = quality_number
+        self._real_quality_number: Optional[QualityNumber] = None
+        self._api_platform: ApiPlatform = api_platform
+        self._use_alternative_stream: bool = use_alternative_stream
+        self._cancelled: bool = False
+
+    def reset(self) -> None:
+        self._real_quality_number = None
+        self._api_platform = 'web'
+        self._use_alternative_stream = False
+        self._cancelled = False
+
+    def cancel(self) -> None:
+        self._cancelled = True
+
+    @property
+    def stream_format(self) -> StreamFormat:
+        return self._stream_format
+
+    @property
+    def quality_number(self) -> QualityNumber:
+        return self._quality_number
+
+    @quality_number.setter
+    def quality_number(self, value: QualityNumber) -> None:
+        self._quality_number = value
+
+    @property
+    def real_quality_number(self) -> QualityNumber:
+        return self._real_quality_number or self._quality_number
+
+    @property
+    def use_alternative_stream(self) -> bool:
+        return self._use_alternative_stream
+
+    @use_alternative_stream.setter
+    def use_alternative_stream(self, value: bool) -> None:
+        self._use_alternative_stream = value
+
+    def fall_back_quality(self) -> None:
+        self._real_quality_number = 10000
+
+    def rotate_api_platform(self) -> None:
+        if self._api_platform == 'android':
+            self._api_platform = 'web'
+        else:
+            self._api_platform = 'android'
+
+    def get_stream_params(self) -> Observable[StreamParams]:
+        self.reset()
+
+        def subscribe(
+            observer: abc.ObserverBase[StreamParams],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            _scheduler = scheduler or CurrentThreadScheduler.singleton()
+
+            disposed = False
+            cancelable = SerialDisposable()
+
+            def action(
+                scheduler: abc.SchedulerBase, state: Optional[Any] = None
+            ) -> None:
+                if self._cancelled or disposed:
+                    return
+
+                params = StreamParams(
+                    stream_format=self._stream_format,
+                    quality_number=self._real_quality_number or self._quality_number,
+                    api_platform=self._api_platform,
+                    use_alternative_stream=self._use_alternative_stream,
+                )
+                observer.on_next(params)
+
+            cancelable.disposable = _scheduler.schedule(action)
+
+            def dispose() -> None:
+                nonlocal disposed
+                disposed = True
+
+            return CompositeDisposable(cancelable, Disposable(dispose))
+
+        return create(subscribe)

src/blrec/core/stream_recorder.py

@@ -0,0 +1,283 @@
+import asyncio
+import logging
+import time
+from typing import Iterator, Optional
+
+from ..bili.live import Live
+from ..bili.typing import QualityNumber, StreamFormat
+from ..event.event_emitter import EventEmitter
+from ..flv.operators import MetaData, StreamProfile
+from ..utils.mixins import AsyncStoppableMixin
+from .flv_stream_recorder_impl import FLVStreamRecorderImpl
+from .hls_stream_recorder_impl import HLSStreamRecorderImpl
+from .stream_recorder_impl import StreamRecorderEventListener
+
+__all__ = 'StreamRecorder', 'StreamRecorderEventListener'
+
+
+logger = logging.getLogger(__name__)
+
+
+class StreamRecorder(
+    StreamRecorderEventListener,
+    EventEmitter[StreamRecorderEventListener],
+    AsyncStoppableMixin,
+):
+    def __init__(
+        self,
+        live: Live,
+        out_dir: str,
+        path_template: str,
+        *,
+        stream_format: StreamFormat = 'flv',
+        quality_number: QualityNumber = 10000,
+        fmp4_stream_timeout: int = 10,
+        buffer_size: Optional[int] = None,
+        read_timeout: Optional[int] = None,
+        disconnection_timeout: Optional[int] = None,
+        filesize_limit: int = 0,
+        duration_limit: int = 0,
+    ) -> None:
+        super().__init__()
+
+        self.stream_format = stream_format
+        self.fmp4_stream_timeout = fmp4_stream_timeout
+
+        if stream_format == 'flv':
+            cls = FLVStreamRecorderImpl
+        elif stream_format == 'fmp4':
+            cls = HLSStreamRecorderImpl  # type: ignore
+        else:
+            logger.warning(
+                f'The specified stream format ({stream_format}) is '
+                'unsupported, will using the stream format (flv) instead.'
+            )
+            self.stream_format = 'flv'
+            cls = FLVStreamRecorderImpl
+
+        self._impl = cls(
+            live=live,
+            out_dir=out_dir,
+            path_template=path_template,
+            quality_number=quality_number,
+            buffer_size=buffer_size,
+            read_timeout=read_timeout,
+            disconnection_timeout=disconnection_timeout,
+            filesize_limit=filesize_limit,
+            duration_limit=duration_limit,
+        )
+
+        self._impl.add_listener(self)
+
+    @property
+    def stream_url(self) -> str:
+        return self._impl.stream_url
+
+    @property
+    def stream_host(self) -> str:
+        return self._impl.stream_host
+
+    @property
+    def dl_total(self) -> int:
+        return self._impl.dl_total
+
+    @property
+    def dl_rate(self) -> float:
+        return self._impl.dl_rate
+
+    @property
+    def rec_elapsed(self) -> float:
+        return self._impl.rec_elapsed
+
+    @property
+    def rec_total(self) -> int:
+        return self._impl.rec_total
+
+    @property
+    def rec_rate(self) -> float:
+        return self._impl.rec_rate
+
+    @property
+    def out_dir(self) -> str:
+        return self._impl.out_dir
+
+    @out_dir.setter
+    def out_dir(self, value: str) -> None:
+        self._impl.out_dir = value
+
+    @property
+    def path_template(self) -> str:
+        return self._impl.path_template
+
+    @path_template.setter
+    def path_template(self, value: str) -> None:
+        self._impl.path_template = value
+
+    @property
+    def quality_number(self) -> QualityNumber:
+        return self._impl.quality_number
+
+    @quality_number.setter
+    def quality_number(self, value: QualityNumber) -> None:
+        self._impl.quality_number = value
+
+    @property
+    def real_stream_format(self) -> Optional[StreamFormat]:
+        if self.stopped:
+            return None
+        return self._impl.stream_format
+
+    @property
+    def real_quality_number(self) -> Optional[QualityNumber]:
+        return self._impl.real_quality_number
+
+    @property
+    def buffer_size(self) -> int:
+        return self._impl.buffer_size
+
+    @buffer_size.setter
+    def buffer_size(self, value: int) -> None:
+        self._impl.buffer_size = value
+
+    @property
+    def read_timeout(self) -> int:
+        return self._impl.read_timeout
+
+    @read_timeout.setter
+    def read_timeout(self, value: int) -> None:
+        self._impl.read_timeout = value
+
+    @property
+    def disconnection_timeout(self) -> int:
+        return self._impl.disconnection_timeout
+
+    @disconnection_timeout.setter
+    def disconnection_timeout(self, value: int) -> None:
+        self._impl.disconnection_timeout = value
+
+    @property
+    def filesize_limit(self) -> int:
+        return self._impl.filesize_limit
+
+    @filesize_limit.setter
+    def filesize_limit(self, value: int) -> None:
+        self._impl.filesize_limit = value
+
+    @property
+    def duration_limit(self) -> int:
+        return self._impl.duration_limit
+
+    @duration_limit.setter
+    def duration_limit(self, value: int) -> None:
+        self._impl.duration_limit = value
+
+    @property
+    def recording_path(self) -> Optional[str]:
+        return self._impl.recording_path
+
+    @property
+    def metadata(self) -> Optional[MetaData]:
+        return self._impl.metadata
+
+    @property
+    def stream_profile(self) -> StreamProfile:
+        return self._impl.stream_profile
+
+    def has_file(self) -> bool:
+        return self._impl.has_file()
+
+    def get_files(self) -> Iterator[str]:
+        yield from self._impl.get_files()
+
+    def clear_files(self) -> None:
+        self._impl.clear_files()
+
+    def can_cut_stream(self) -> bool:
+        return self._impl.can_cut_stream()
+
+    def cut_stream(self) -> bool:
+        return self._impl.cut_stream()
+
+    def update_progress_bar_info(self) -> None:
+        self._impl.update_progress_bar_info()
+
+    @property
+    def stopped(self) -> bool:
+        return self._impl.stopped
+
+    async def _do_start(self) -> None:
+        stream_format = self.stream_format
+        if stream_format == 'fmp4':
+            logger.info('Waiting for the fmp4 stream becomes available...')
+            available = await self._wait_fmp4_stream()
+            if not available:
+                logger.warning(
+                    'The specified stream format (fmp4) is not available '
+                    f'in {self.fmp4_stream_timeout} seconcds, '
+                    'falling back to stream format (flv).'
+                )
+                stream_format = 'flv'
+        self._change_impl(stream_format)
+        await self._impl.start()
+
+    async def _do_stop(self) -> None:
+        await self._impl.stop()
+
+    async def on_video_file_created(self, path: str, record_start_time: int) -> None:
+        await self._emit('video_file_created', path, record_start_time)
+
+    async def on_video_file_completed(self, path: str) -> None:
+        await self._emit('video_file_completed', path)
+
+    async def on_stream_recording_completed(self) -> None:
+        await self._emit('stream_recording_completed')
+
+    async def _wait_fmp4_stream(self) -> bool:
+        end_time = time.monotonic() + self.fmp4_stream_timeout
+        available = False  # debounce
+        while True:
+            try:
+                await self._impl._live.get_live_stream_url(stream_format='fmp4')
+            except Exception:
+                available = False
+                if time.monotonic() > end_time:
+                    return False
+            else:
+                if available:
+                    return True
+                else:
+                    available = True
+            await asyncio.sleep(1)
+
+    def _change_impl(self, stream_format: StreamFormat) -> None:
+        if stream_format == 'flv':
+            cls = FLVStreamRecorderImpl
+        elif stream_format == 'fmp4':
+            cls = HLSStreamRecorderImpl  # type: ignore
+        else:
+            logger.warning(
+                f'The specified stream format ({stream_format}) is '
+                'unsupported, will using the stream format (flv) instead.'
+            )
+            cls = FLVStreamRecorderImpl
+
+        if self._impl.__class__ == cls:
+            return
+
+        self._impl.remove_listener(self)
+
+        self._impl = cls(
+            live=self._impl._live,
+            out_dir=self._impl.out_dir,
+            path_template=self._impl.path_template,
+            quality_number=self._impl.quality_number,
+            buffer_size=self._impl.buffer_size,
+            read_timeout=self._impl.read_timeout,
+            disconnection_timeout=self._impl.disconnection_timeout,
+            filesize_limit=self._impl.filesize_limit,
+            duration_limit=self._impl.duration_limit,
+        )
+
+        self._impl.add_listener(self)
+
+        logger.debug(f'Changed stream recorder impl to {cls.__name__}')

src/blrec/core/stream_recorder_impl.py

@@ -0,0 +1,307 @@
+import logging
+from abc import ABC, abstractmethod
+from threading import Thread
+from typing import Any, Iterator, List, Optional, Tuple, Union
+
+import requests
+import urllib3
+from reactivex import abc
+from reactivex.typing import StartableFactory, StartableTarget
+
+from ..bili.live import Live
+from ..bili.typing import QualityNumber, StreamFormat
+from ..event.event_emitter import EventEmitter, EventListener
+from ..flv import operators as flv_ops
+from ..flv.metadata_dumper import MetadataDumper
+from ..flv.operators import StreamProfile
+from ..logging.room_id import aio_task_with_room_id
+from ..utils.mixins import AsyncCooperationMixin, AsyncStoppableMixin
+from . import operators as core_ops
+from .metadata_provider import MetadataProvider
+from .path_provider import PathProvider
+from .stream_param_holder import StreamParamHolder
+
+__all__ = ('StreamRecorderImpl',)
+
+
+logger = logging.getLogger(__name__)
+logging.getLogger(urllib3.__name__).setLevel(logging.WARNING)
+
+
+class StreamRecorderEventListener(EventListener):
+    async def on_video_file_created(self, path: str, record_start_time: int) -> None:
+        ...
+
+    async def on_video_file_completed(self, path: str) -> None:
+        ...
+
+    async def on_stream_recording_completed(self) -> None:
+        ...
+
+
+class StreamRecorderImpl(
+    EventEmitter[StreamRecorderEventListener],
+    AsyncCooperationMixin,
+    AsyncStoppableMixin,
+    ABC,
+):
+    def __init__(
+        self,
+        live: Live,
+        out_dir: str,
+        path_template: str,
+        *,
+        stream_format: StreamFormat = 'flv',
+        quality_number: QualityNumber = 10000,
+        buffer_size: Optional[int] = None,
+        read_timeout: Optional[int] = None,
+        disconnection_timeout: Optional[int] = None,
+        filesize_limit: int = 0,
+        duration_limit: int = 0,
+    ) -> None:
+        super().__init__()
+
+        self._live = live
+        self._session = requests.Session()
+        self._stream_param_holder = StreamParamHolder(
+            stream_format=stream_format, quality_number=quality_number
+        )
+        self._stream_url_resolver = core_ops.StreamURLResolver(
+            live, self._stream_param_holder
+        )
+        self._stream_fetcher = core_ops.StreamFetcher(
+            live, self._session, read_timeout=read_timeout
+        )
+        self._stream_parser = core_ops.StreamParser(
+            self._stream_param_holder, ignore_eof=stream_format != 'flv'
+        )
+        self._progress_bar = core_ops.ProgressBar(live)
+        self._analyser = flv_ops.Analyser()
+        self._metadata_provider = MetadataProvider(live, self)
+        self._injector = flv_ops.Injector(self._metadata_provider)
+        self._join_point_extractor = flv_ops.JoinPointExtractor()
+        self._limiter = flv_ops.Limiter(filesize_limit, duration_limit)
+        self._cutter = flv_ops.Cutter()
+        self._path_provider = PathProvider(live, out_dir, path_template)
+        self._dumper = flv_ops.Dumper(self._path_provider, buffer_size)
+        self._rec_statistics = core_ops.SizedStatistics()
+
+        self._prober: Union[flv_ops.Prober, core_ops.HLSProber]
+        self._dl_statistics: Union[core_ops.StreamStatistics, core_ops.SizedStatistics]
+        if stream_format == 'flv':
+            self._prober = flv_ops.Prober()
+            self._dl_statistics = core_ops.StreamStatistics()
+        else:
+            self._prober = core_ops.HLSProber()
+            self._dl_statistics = core_ops.SizedStatistics()
+
+        self._request_exception_handler = core_ops.RequestExceptionHandler()
+        self._connection_error_handler = core_ops.ConnectionErrorHandler(
+            live, disconnection_timeout=disconnection_timeout
+        )
+        self._exception_handler = core_ops.ExceptionHandler()
+        self._metadata_dumper = MetadataDumper(
+            self._dumper, self._analyser, self._join_point_extractor
+        )
+        self._metadata_dumper.enable()
+
+        self._subscription: abc.DisposableBase
+
+        self._threads: List[Thread] = []
+        self._files: List[str] = []
+        self._stream_profile: StreamProfile = {}
+
+        def on_profile_updated(profile: StreamProfile) -> None:
+            self._stream_profile = profile
+
+        self._prober.profiles.subscribe(on_profile_updated)
+
+        def on_file_opened(args: Tuple[str, int]) -> None:
+            logger.info(f"Video file created: '{args[0]}'")
+            self._files.append(args[0])
+            self._emit_event('video_file_created', *args)
+
+        def on_file_closed(path: str) -> None:
+            logger.info(f"Video file completed: '{path}'")
+            self._emit_event('video_file_completed', path)
+
+        self._dumper.file_opened.subscribe(on_file_opened)
+        self._dumper.file_closed.subscribe(on_file_closed)
+
+    @property
+    def stream_url(self) -> str:
+        return self._stream_url_resolver.stream_url
+
+    @property
+    def stream_host(self) -> str:
+        return self._stream_url_resolver.stream_host
+
+    @property
+    def dl_total(self) -> int:
+        return self._dl_statistics.count
+
+    @property
+    def dl_rate(self) -> float:
+        return self._dl_statistics.rate
+
+    @property
+    def rec_elapsed(self) -> float:
+        return self._rec_statistics.elapsed
+
+    @property
+    def rec_total(self) -> int:
+        return self._rec_statistics.count
+
+    @property
+    def rec_rate(self) -> float:
+        return self._rec_statistics.rate
+
+    @property
+    def out_dir(self) -> str:
+        return self._path_provider.out_dir
+
+    @out_dir.setter
+    def out_dir(self, value: str) -> None:
+        self._path_provider.out_dir = value
+
+    @property
+    def path_template(self) -> str:
+        return self._path_provider.path_template
+
+    @path_template.setter
+    def path_template(self, value: str) -> None:
+        self._path_provider.path_template = value
+
+    @property
+    def stream_format(self) -> StreamFormat:
+        return self._stream_param_holder.stream_format
+
+    @property
+    def quality_number(self) -> QualityNumber:
+        return self._stream_param_holder.quality_number
+
+    @quality_number.setter
+    def quality_number(self, value: QualityNumber) -> None:
+        self._stream_param_holder.quality_number = value
+
+    @property
+    def real_quality_number(self) -> Optional[QualityNumber]:
+        if self.stopped:
+            return None
+        return self._stream_param_holder.real_quality_number
+
+    @property
+    def filesize_limit(self) -> int:
+        return self._limiter.filesize_limit
+
+    @filesize_limit.setter
+    def filesize_limit(self, value: int) -> None:
+        self._limiter.filesize_limit = value
+
+    @property
+    def duration_limit(self) -> int:
+        return self._limiter.duration_limit
+
+    @duration_limit.setter
+    def duration_limit(self, value: int) -> None:
+        self._limiter.duration_limit = value
+
+    @property
+    def read_timeout(self) -> int:
+        return self._stream_fetcher.read_timeout
+
+    @read_timeout.setter
+    def read_timeout(self, value: int) -> None:
+        self._stream_fetcher.read_timeout = value
+
+    @property
+    def disconnection_timeout(self) -> int:
+        return self._connection_error_handler.disconnection_timeout
+
+    @disconnection_timeout.setter
+    def disconnection_timeout(self, value: int) -> None:
+        self._connection_error_handler.disconnection_timeout = value
+
+    @property
+    def buffer_size(self) -> int:
+        return self._dumper.buffer_size
+
+    @buffer_size.setter
+    def buffer_size(self, value: int) -> None:
+        self._dumper.buffer_size = value
+
+    @property
+    def recording_path(self) -> Optional[str]:
+        return self._dumper.path
+
+    @property
+    def metadata(self) -> Optional[flv_ops.MetaData]:
+        try:
+            return self._analyser.make_metadata()
+        except Exception:
+            return None
+
+    @property
+    def stream_profile(self) -> StreamProfile:
+        return self._stream_profile
+
+    def has_file(self) -> bool:
+        return bool(self._files)
+
+    def get_files(self) -> Iterator[str]:
+        yield from iter(self._files)
+
+    def clear_files(self) -> None:
+        self._files.clear()
+
+    def can_cut_stream(self) -> bool:
+        return self._cutter.can_cut_stream()
+
+    def cut_stream(self) -> bool:
+        return self._cutter.cut_stream()
+
+    def update_progress_bar_info(self) -> None:
+        self._progress_bar.update_bar_info()
+
+    def _reset(self) -> None:
+        self._files.clear()
+        self._stream_profile = {}
+
+    async def _do_start(self) -> None:
+        logger.debug('Starting stream recorder...')
+        self._reset()
+        self._run()
+        logger.debug('Started stream recorder')
+
+    async def _do_stop(self) -> None:
+        logger.debug('Stopping stream recorder...')
+        self._stream_param_holder.cancel()
+        thread = self._thread_factory('StreamRecorderDisposer')(
+            self._subscription.dispose
+        )
+        thread.start()
+        for thread in self._threads:
+            await self._loop.run_in_executor(None, thread.join, 30)
+        self._threads.clear()
+        logger.debug('Stopped stream recorder')
+
+    @abstractmethod
+    def _run(self) -> None:
+        raise NotImplementedError()
+
+    def _thread_factory(self, name: str) -> StartableFactory:
+        def factory(target: StartableTarget) -> Thread:
+            thread = Thread(
+                target=target, daemon=True, name=f'{name}::{self._live.room_id}'
+            )
+            self._threads.append(thread)
+            return thread
+
+        return factory
+
+    def _emit_event(self, name: str, *args: Any, **kwds: Any) -> None:
+        self._run_coroutine(self._emit(name, *args, **kwds))
+
+    @aio_task_with_room_id
+    async def _emit(self, *args: Any, **kwds: Any) -> None:  # type: ignore
+        await super()._emit(*args, **kwds)

src/blrec/core/stream_remuxer.py

@@ -1,31 +1,37 @@
-import os
-import io
 import errno
-import shlex
+import io
 import logging
-from threading import Thread, Event
-from subprocess import Popen, PIPE, CalledProcessError
-from typing import List, Optional, cast
-
+import os
+import re
+import shlex
+from subprocess import PIPE, CalledProcessError, Popen
+from threading import Condition, Thread
+from typing import Optional, cast
 
+from ..utils.io import wait_for
 from ..utils.mixins import StoppableMixin, SupportDebugMixin
 
-
 logger = logging.getLogger(__name__)
 
 
-__all__ = 'StreamRemuxer',
+__all__ = ('StreamRemuxer',)
+
+
+class FFmpegError(Exception):
+    pass
 
 
 class StreamRemuxer(StoppableMixin, SupportDebugMixin):
-    def __init__(self, room_id: int, bufsize: int = 1024 * 1024) -> None:
+    _ERROR_PATTERN = re.compile(
+        r'\b(error|failed|missing|invalid|corrupt)\b', re.IGNORECASE
+    )
+
+    def __init__(self, room_id: int, remove_filler_data: bool = False) -> None:
         super().__init__()
         self._room_id = room_id
-        self._bufsize = bufsize
+        self._remove_filler_data = remove_filler_data
         self._exception: Optional[Exception] = None
-        self._subprocess_setup = Event()
-        self._MAX_ERROR_MESSAGES = 10
-        self._error_messages: List[str] = []
+        self._ready = Condition()
         self._env = None
 
         self._init_for_debug(room_id)
@@ -50,15 +56,22 @@ class StreamRemuxer(StoppableMixin, SupportDebugMixin):
 
     def __enter__(self):  # type: ignore
         self.start()
-        self.wait_for_subprocess()
+        self.wait()
         return self
 
     def __exit__(self, exc_type, value, traceback):  # type: ignore
         self.stop()
         self.raise_for_exception()
 
-    def wait_for_subprocess(self) -> None:
-        self._subprocess_setup.wait()
+    def wait(self, timeout: Optional[float] = None) -> bool:
+        with self._ready:
+            return self._ready.wait(timeout=timeout)
+
+    def restart(self) -> None:
+        logger.debug('Restarting stream remuxer...')
+        self.stop()
+        self.start()
+        logger.debug('Restarted stream remuxer')
 
     def raise_for_exception(self) -> None:
         if not self.exception:
@@ -68,9 +81,7 @@ class StreamRemuxer(StoppableMixin, SupportDebugMixin):
     def _do_start(self) -> None:
         logger.debug('Starting stream remuxer...')
         self._thread = Thread(
-            target=self._run,
-            name=f'StreamRemuxer::{self._room_id}',
-            daemon=True,
+            target=self._run, name=f'StreamRemuxer::{self._room_id}', daemon=True
         )
         self._thread.start()
 
@@ -85,58 +96,63 @@ class StreamRemuxer(StoppableMixin, SupportDebugMixin):
     def _run(self) -> None:
         logger.debug('Started stream remuxer')
         self._exception = None
-        self._error_messages.clear()
-        self._subprocess_setup.clear()
         try:
             self._run_subprocess()
-        except BrokenPipeError:
-            pass
-        except Exception as e:
+        except BrokenPipeError as exc:
+            logger.debug(repr(exc))
+        except FFmpegError as exc:
+            if not self._stopped:
+                logger.warning(repr(exc))
+            else:
+                logger.debug(repr(exc))
+        except TimeoutError as exc:
+            logger.debug(repr(exc))
+        except Exception as exc:
             # OSError: [Errno 22] Invalid argument
             # https://stackoverflow.com/questions/23688492/oserror-errno-22-invalid-argument-in-subprocess
-            if isinstance(e, OSError) and e.errno == errno.EINVAL:
+            if isinstance(exc, OSError) and exc.errno == errno.EINVAL:
                 pass
             else:
-                self._exception = e
-                logger.exception(e)
+                self._exception = exc
+                logger.exception(exc)
         finally:
+            self._stopped = True
             logger.debug('Stopped stream remuxer')
 
     def _run_subprocess(self) -> None:
-        cmd = 'ffmpeg -i pipe:0 -c copy -f flv pipe:1'
+        cmd = 'ffmpeg -xerror -i pipe:0 -c copy -copyts'
+        if self._remove_filler_data:
+            cmd += ' -bsf:v filter_units=remove_types=12'
+        cmd += ' -f flv pipe:1'
         args = shlex.split(cmd)
 
         with Popen(
-            args, stdin=PIPE, stdout=PIPE, stderr=PIPE,
-            bufsize=self._bufsize, env=self._env,
+            args, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=self._env
         ) as self._subprocess:
-            self._subprocess_setup.set()
-            assert self._subprocess.stderr is not None
+            with self._ready:
+                self._ready.notify_all()
 
-            while not self._stopped:
-                data = self._subprocess.stderr.readline()
-                if not data:
-                    if self._subprocess.poll() is not None:
-                        break
-                    else:
-                        continue
-                line = data.decode('utf-8', errors='backslashreplace')
-                if self._debug:
-                    logger.debug('ffmpeg: %s', line)
-                self._check_error(line)
+            assert self._subprocess.stderr is not None
+            with io.TextIOWrapper(
+                self._subprocess.stderr, encoding='utf-8', errors='backslashreplace'
+            ) as stderr:
+                while not self._stopped:
+                    line = wait_for(stderr.readline, timeout=10)
+                    if not line:
+                        if self._subprocess.poll() is not None:
+                            break
+                        else:
+                            continue
+                    if self._debug:
+                        logger.debug('ffmpeg: %s', line)
+                    self._check_error(line)
 
         if not self._stopped and self._subprocess.returncode not in (0, 255):
             # 255: Exiting normally, received signal 2.
-            raise CalledProcessError(
-                self._subprocess.returncode,
-                cmd=cmd,
-                output='\n'.join(self._error_messages),
-            )
+            raise CalledProcessError(self._subprocess.returncode, cmd=cmd)
 
     def _check_error(self, line: str) -> None:
-        if 'error' not in line.lower() and 'failed' not in line.lower():
+        match = self._ERROR_PATTERN.search(line)
+        if not match:
             return
-        logger.warning(f'ffmpeg error: {line}')
-        self._error_messages.append(line)
-        if len(self._error_messages) > self._MAX_ERROR_MESSAGES:
-            self._error_messages.remove(self._error_messages[0])
+        raise FFmpegError(line)

src/blrec/data/webapp/index.html

@@ -10,6 +10,6 @@
 <body>
   <app-root></app-root>
   <noscript>Please enable JavaScript to continue using this application.</noscript>
-<script src="runtime.23c91f03d62c595a.js" type="module"></script><script src="polyfills.4b08448aee19bb22.js" type="module"></script><script src="main.8a8c73fae6ff9291.js" type="module"></script>
+<script src="runtime.8ba8344712d0946d.js" type="module"></script><script src="polyfills.4b08448aee19bb22.js" type="module"></script><script src="main.b9234f0840c7101a.js" type="module"></script>
 
 </body></html>

src/blrec/data/webapp/ngsw.json

@@ -1,6 +1,6 @@
 {
   "configVersion": 1,
-  "timestamp": 1649386979751,
+  "timestamp": 1653113686866,
   "index": "/index.html",
   "assetGroups": [
     {
@@ -14,15 +14,15 @@
         "/103.5b5d2a6e5a8a7479.js",
         "/146.92e3b29c4c754544.js",
         "/45.c90c3cea2bf1a66e.js",
-        "/66.d8b06f1fef317761.js",
-        "/694.92a3e0c2fc842a42.js",
-        "/869.95d68b28a4188d76.js",
+        "/474.7f6529972e383566.js",
+        "/66.31f5b9ae46ae9005.js",
+        "/869.0ab6b8a3f466df77.js",
         "/common.858f777e9296e6f2.js",
         "/index.html",
-        "/main.8a8c73fae6ff9291.js",
+        "/main.b9234f0840c7101a.js",
         "/manifest.webmanifest",
         "/polyfills.4b08448aee19bb22.js",
-        "/runtime.23c91f03d62c595a.js",
+        "/runtime.8ba8344712d0946d.js",
         "/styles.1f581691b230dc4d.css"
       ],
       "patterns": []
@@ -1637,9 +1637,9 @@
     "/103.5b5d2a6e5a8a7479.js": "cc0240f217015b6d4ddcc14f31fcc42e1c1c282a",
     "/146.92e3b29c4c754544.js": "3824de681dd1f982ea69a065cdf54d7a1e781f4d",
     "/45.c90c3cea2bf1a66e.js": "e5bfb8cf3803593e6b8ea14c90b3d3cb6a066764",
-    "/66.d8b06f1fef317761.js": "43676d9dc886b5624dadecc50f17d4972b183d2d",
-    "/694.92a3e0c2fc842a42.js": "f8f093029b9996b3db0c4e738bf9f8573fba8392",
-    "/869.95d68b28a4188d76.js": "cd1add38c89b1df3c0783b74c931b51839f1c530",
+    "/474.7f6529972e383566.js": "1c74b5c6379705a3110c99767f97feddc42a0d54",
+    "/66.31f5b9ae46ae9005.js": "cc22d2582d8e4c2a83e089d5a1ec32619e439ccd",
+    "/869.0ab6b8a3f466df77.js": "fd3e32d78790ec916177aa38b49ca32bfe62d0d4",
     "/assets/animal/panda.js": "fec2868bb3053dd2da45f96bbcb86d5116ed72b1",
     "/assets/animal/panda.svg": "bebd302cdc601e0ead3a6d2710acf8753f3d83b1",
     "/assets/fill/.gitkeep": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
@@ -3234,11 +3234,11 @@
     "/assets/twotone/warning.js": "fb2d7ea232f3a99bf8f080dbc94c65699232ac01",
     "/assets/twotone/warning.svg": "8c7a2d3e765a2e7dd58ac674870c6655cecb0068",
     "/common.858f777e9296e6f2.js": "b68ca68e1e214a2537d96935c23410126cc564dd",
-    "/index.html": "114f00ffcd1f7fa5aaaa7f2fcf3109f26c77c715",
-    "/main.8a8c73fae6ff9291.js": "41a5a5a8fb5cda4cfa0e28532812594816257122",
+    "/index.html": "da07776fe6f05347b1b8360f61b81c59734dcc54",
+    "/main.b9234f0840c7101a.js": "c8c7b588c070b957a2659f62d6a77de284aa2233",
     "/manifest.webmanifest": "62c1cb8c5ad2af551a956b97013ab55ce77dd586",
     "/polyfills.4b08448aee19bb22.js": "8e73f2d42cc13ca353cea5c886d930bd6da08d0d",
-    "/runtime.23c91f03d62c595a.js": "0819f1120ed1e37c2ad069ef949147450c951069",
+    "/runtime.8ba8344712d0946d.js": "264d0e7e1e88dd1a4383d73d401f5ccd51e40eb7",
     "/styles.1f581691b230dc4d.css": "6f5befbbad57c2b2e80aae855139744b8010d150"
   },
   "navigationUrls": [

src/blrec/data/webapp/runtime.8ba8344712d0946d.js

@@ -1 +1 @@
-(()=>{"use strict";var e,v={},m={};function r(e){var i=m[e];if(void 0!==i)return i.exports;var t=m[e]={exports:{}};return v[e].call(t.exports,t,t.exports,r),t.exports}r.m=v,e=[],r.O=(i,t,f,o)=>{if(!t){var a=1/0;for(n=0;n<e.length;n++){for(var[t,f,o]=e[n],c=!0,l=0;l<t.length;l++)(!1&o||a>=o)&&Object.keys(r.O).every(b=>r.O[b](t[l]))?t.splice(l--,1):(c=!1,o<a&&(a=o));if(c){e.splice(n--,1);var d=f();void 0!==d&&(i=d)}}return i}o=o||0;for(var n=e.length;n>0&&e[n-1][2]>o;n--)e[n]=e[n-1];e[n]=[t,f,o]},r.n=e=>{var i=e&&e.__esModule?()=>e.default:()=>e;return r.d(i,{a:i}),i},r.d=(e,i)=>{for(var t in i)r.o(i,t)&&!r.o(e,t)&&Object.defineProperty(e,t,{enumerable:!0,get:i[t]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce((i,t)=>(r.f[t](e,i),i),[])),r.u=e=>(592===e?"common":e)+"."+{45:"c90c3cea2bf1a66e",66:"d8b06f1fef317761",103:"5b5d2a6e5a8a7479",146:"92e3b29c4c754544",592:"858f777e9296e6f2",694:"92a3e0c2fc842a42",869:"95d68b28a4188d76"}[e]+".js",r.miniCssF=e=>{},r.o=(e,i)=>Object.prototype.hasOwnProperty.call(e,i),(()=>{var e={},i="blrec:";r.l=(t,f,o,n)=>{if(e[t])e[t].push(f);else{var a,c;if(void 0!==o)for(var l=document.getElementsByTagName("script"),d=0;d<l.length;d++){var u=l[d];if(u.getAttribute("src")==t||u.getAttribute("data-webpack")==i+o){a=u;break}}a||(c=!0,(a=document.createElement("script")).type="module",a.charset="utf-8",a.timeout=120,r.nc&&a.setAttribute("nonce",r.nc),a.setAttribute("data-webpack",i+o),a.src=r.tu(t)),e[t]=[f];var s=(g,b)=>{a.onerror=a.onload=null,clearTimeout(p);var _=e[t];if(delete e[t],a.parentNode&&a.parentNode.removeChild(a),_&&_.forEach(h=>h(b)),g)return g(b)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:a}),12e4);a.onerror=s.bind(null,a.onerror),a.onload=s.bind(null,a.onload),c&&document.head.appendChild(a)}}})(),r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},(()=>{var e;r.tu=i=>(void 0===e&&(e={createScriptURL:t=>t},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(e=trustedTypes.createPolicy("angular#bundler",e))),e.createScriptURL(i))})(),r.p="",(()=>{var e={666:0};r.f.j=(f,o)=>{var n=r.o(e,f)?e[f]:void 0;if(0!==n)if(n)o.push(n[2]);else if(666!=f){var a=new Promise((u,s)=>n=e[f]=[u,s]);o.push(n[2]=a);var c=r.p+r.u(f),l=new Error;r.l(c,u=>{if(r.o(e,f)&&(0!==(n=e[f])&&(e[f]=void 0),n)){var s=u&&("load"===u.type?"missing":u.type),p=u&&u.target&&u.target.src;l.message="Loading chunk "+f+" failed.\n("+s+": "+p+")",l.name="ChunkLoadError",l.type=s,l.request=p,n[1](l)}},"chunk-"+f,f)}else e[f]=0},r.O.j=f=>0===e[f];var i=(f,o)=>{var l,d,[n,a,c]=o,u=0;if(n.some(p=>0!==e[p])){for(l in a)r.o(a,l)&&(r.m[l]=a[l]);if(c)var s=c(r)}for(f&&f(o);u<n.length;u++)r.o(e,d=n[u])&&e[d]&&e[d][0](),e[n[u]]=0;return r.O(s)},t=self.webpackChunkblrec=self.webpackChunkblrec||[];t.forEach(i.bind(null,0)),t.push=i.bind(null,t.push.bind(t))})()})();
+(()=>{"use strict";var e,v={},m={};function r(e){var i=m[e];if(void 0!==i)return i.exports;var t=m[e]={exports:{}};return v[e].call(t.exports,t,t.exports,r),t.exports}r.m=v,e=[],r.O=(i,t,o,f)=>{if(!t){var a=1/0;for(n=0;n<e.length;n++){for(var[t,o,f]=e[n],c=!0,l=0;l<t.length;l++)(!1&f||a>=f)&&Object.keys(r.O).every(b=>r.O[b](t[l]))?t.splice(l--,1):(c=!1,f<a&&(a=f));if(c){e.splice(n--,1);var d=o();void 0!==d&&(i=d)}}return i}f=f||0;for(var n=e.length;n>0&&e[n-1][2]>f;n--)e[n]=e[n-1];e[n]=[t,o,f]},r.n=e=>{var i=e&&e.__esModule?()=>e.default:()=>e;return r.d(i,{a:i}),i},r.d=(e,i)=>{for(var t in i)r.o(i,t)&&!r.o(e,t)&&Object.defineProperty(e,t,{enumerable:!0,get:i[t]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce((i,t)=>(r.f[t](e,i),i),[])),r.u=e=>(592===e?"common":e)+"."+{45:"c90c3cea2bf1a66e",66:"31f5b9ae46ae9005",103:"5b5d2a6e5a8a7479",146:"92e3b29c4c754544",474:"7f6529972e383566",592:"858f777e9296e6f2",869:"0ab6b8a3f466df77"}[e]+".js",r.miniCssF=e=>{},r.o=(e,i)=>Object.prototype.hasOwnProperty.call(e,i),(()=>{var e={},i="blrec:";r.l=(t,o,f,n)=>{if(e[t])e[t].push(o);else{var a,c;if(void 0!==f)for(var l=document.getElementsByTagName("script"),d=0;d<l.length;d++){var u=l[d];if(u.getAttribute("src")==t||u.getAttribute("data-webpack")==i+f){a=u;break}}a||(c=!0,(a=document.createElement("script")).type="module",a.charset="utf-8",a.timeout=120,r.nc&&a.setAttribute("nonce",r.nc),a.setAttribute("data-webpack",i+f),a.src=r.tu(t)),e[t]=[o];var s=(g,b)=>{a.onerror=a.onload=null,clearTimeout(p);var _=e[t];if(delete e[t],a.parentNode&&a.parentNode.removeChild(a),_&&_.forEach(h=>h(b)),g)return g(b)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:a}),12e4);a.onerror=s.bind(null,a.onerror),a.onload=s.bind(null,a.onload),c&&document.head.appendChild(a)}}})(),r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},(()=>{var e;r.tu=i=>(void 0===e&&(e={createScriptURL:t=>t},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(e=trustedTypes.createPolicy("angular#bundler",e))),e.createScriptURL(i))})(),r.p="",(()=>{var e={666:0};r.f.j=(o,f)=>{var n=r.o(e,o)?e[o]:void 0;if(0!==n)if(n)f.push(n[2]);else if(666!=o){var a=new Promise((u,s)=>n=e[o]=[u,s]);f.push(n[2]=a);var c=r.p+r.u(o),l=new Error;r.l(c,u=>{if(r.o(e,o)&&(0!==(n=e[o])&&(e[o]=void 0),n)){var s=u&&("load"===u.type?"missing":u.type),p=u&&u.target&&u.target.src;l.message="Loading chunk "+o+" failed.\n("+s+": "+p+")",l.name="ChunkLoadError",l.type=s,l.request=p,n[1](l)}},"chunk-"+o,o)}else e[o]=0},r.O.j=o=>0===e[o];var i=(o,f)=>{var l,d,[n,a,c]=f,u=0;if(n.some(p=>0!==e[p])){for(l in a)r.o(a,l)&&(r.m[l]=a[l]);if(c)var s=c(r)}for(o&&o(f);u<n.length;u++)r.o(e,d=n[u])&&e[d]&&e[d][0](),e[n[u]]=0;return r.O(s)},t=self.webpackChunkblrec=self.webpackChunkblrec||[];t.forEach(i.bind(null,0)),t.push=i.bind(null,t.push.bind(t))})()})();

src/blrec/disk_space/space_monitor.py

@@ -1,16 +1,14 @@
-import logging
 import asyncio
+import logging
 import shutil
 from contextlib import suppress
 
-
-from .models import DiskUsage
-from .helpers import is_space_enough
-from ..event.event_emitter import EventListener, EventEmitter
+from ..event.event_emitter import EventEmitter, EventListener
 from ..exception import exception_callback
-from ..utils.mixins import SwitchableMixin
 from ..logging.room_id import aio_task_with_room_id
-
+from ..utils.mixins import AsyncStoppableMixin, SwitchableMixin
+from .helpers import is_space_enough
+from .models import DiskUsage
 
 __all__ = 'SpaceMonitor', 'SpaceEventListener'
 
@@ -25,7 +23,9 @@ class SpaceEventListener(EventListener):
         ...
 
 
-class SpaceMonitor(EventEmitter[SpaceEventListener], SwitchableMixin):
+class SpaceMonitor(
+    EventEmitter[SpaceEventListener], SwitchableMixin, AsyncStoppableMixin
+):
     def __init__(
         self,
         path: str,
@@ -40,24 +40,18 @@ class SpaceMonitor(EventEmitter[SpaceEventListener], SwitchableMixin):
         self._monitoring: bool = False
 
     def _do_enable(self) -> None:
-        asyncio.create_task(self._start())
+        asyncio.create_task(self.start())
         logger.debug('Enabled space monitor')
 
     def _do_disable(self) -> None:
-        asyncio.create_task(self._stop())
+        asyncio.create_task(self.stop())
         logger.debug('Disabled space monitor')
 
-    async def _start(self) -> None:
-        if self._monitoring:
-            return
+    async def _do_start(self) -> None:
         self._create_polling_task()
-        self._monitoring = True
 
-    async def _stop(self) -> None:
-        if not self._monitoring:
-            return
+    async def _do_stop(self) -> None:
         await self._cancel_polling_task()
-        self._monitoring = False
 
     def _create_polling_task(self) -> None:
         self._polling_task = asyncio.create_task(self._polling_loop())
@@ -78,6 +72,4 @@ class SpaceMonitor(EventEmitter[SpaceEventListener], SwitchableMixin):
 
     async def _emit_space_no_enough(self) -> None:
         usage = DiskUsage(*shutil.disk_usage(self.path))
-        await self._emit(
-            'space_no_enough', self.path, self.space_threshold, usage
-        )
+        await self._emit('space_no_enough', self.path, self.space_threshold, usage)

src/blrec/disk_space/space_reclaimer.py

@@ -7,19 +7,14 @@ import asyncio
 from functools import partial
 from typing import Iterable, List
 
-from tenacity import (
-    retry,
-    wait_none,
-    stop_after_attempt,
-    retry_if_exception_type,
-)
+from tenacity import retry, wait_none, stop_after_attempt, retry_if_exception_type
 
 from .helpers import delete_file, is_space_enough
 from .space_monitor import SpaceMonitor, DiskUsage, SpaceEventListener
 from ..utils.mixins import SwitchableMixin
 
 
-__all__ = 'SpaceReclaimer',
+__all__ = ('SpaceReclaimer',)
 
 
 logger = logging.getLogger(__name__)
@@ -33,11 +28,18 @@ class SpaceReclaimer(SpaceEventListener, SwitchableMixin):
         space_monitor: SpaceMonitor,
         path: str,
         *,
+        rec_ttl: int = 60 * 60 * 24,
         recycle_records: bool = False,
     ) -> None:
         super().__init__()
         self._space_monitor = space_monitor
         self.path = path
+        if value := os.environ.get('REC_TTL'):
+            try:
+                rec_ttl = int(value)
+            except Exception as exc:
+                logger.warning(repr(exc))
+        self.rec_ttl = rec_ttl
         self.recycle_records = recycle_records
 
     async def on_space_no_enough(
@@ -63,9 +65,8 @@ class SpaceReclaimer(SpaceEventListener, SwitchableMixin):
 
     async def _free_space_from_records(self, size: int) -> bool:
         logger.info('Free space from records ...')
-        # only delete files created 24 hours ago
-        max_ctime = datetime.now().timestamp() - 60 * 60 * 24
-        for path in await self._get_record_file_paths(max_ctime):
+        ts = datetime.now().timestamp() - self.rec_ttl
+        for path in await self._get_record_file_paths(ts):
             await delete_file(path)
             if is_space_enough(self.path, size):
                 return True
@@ -76,13 +77,15 @@ class SpaceReclaimer(SpaceEventListener, SwitchableMixin):
         wait=wait_none(),
         stop=stop_after_attempt(3),
     )
-    async def _get_record_file_paths(self, max_ctime: float) -> List[str]:
+    async def _get_record_file_paths(self, ts: float) -> List[str]:
         glob_path = os.path.join(self.path, '*/**/*.*')
         paths: Iterable[Path]
         paths = map(lambda p: Path(p), glob.iglob(glob_path, recursive=True))
         paths = filter(lambda p: p.suffix in self._SUFFIX_SET, paths)
-        paths = filter(lambda p: p.stat().st_ctime <= max_ctime, paths)
-        func = partial(sorted, paths, key=lambda p: p.stat().st_ctime)
+        paths = filter(lambda p: p.stat().st_mtime < ts > p.stat().st_atime, paths)
+        func = partial(
+            sorted, paths, key=lambda p: (p.stat().st_mtime, p.stat().st_atime)
+        )
         loop = asyncio.get_running_loop()
         path_list = await loop.run_in_executor(None, func)
         return list(map(str, path_list))

src/blrec/event/event_center.py

@@ -1,21 +1,18 @@
+from reactivex import Observable, Subject
 
-from rx.subject import Subject
-from rx.core import Observable
-
-from .typing import Event
 from ..utils.patterns import Singleton
+from .typing import Event
 
-
-__all__ = 'EventCenter',
+__all__ = ('EventCenter',)
 
 
 class EventCenter(Singleton):
     def __init__(self) -> None:
         super().__init__()
-        self._source = Subject()
+        self._source: Subject[Event] = Subject()
 
     @property
-    def events(self) -> Observable:
+    def events(self) -> Observable[Event]:
         return self._source
 
     def submit(self, event: Event) -> None:

src/blrec/exception/exception_center.py

@@ -1,20 +1,17 @@
-
-from rx.subject import Subject
-from rx.core import Observable
+from reactivex import Observable, Subject
 
 from ..utils.patterns import Singleton
 
-
-__all__ = 'ExceptionCenter',
+__all__ = ('ExceptionCenter',)
 
 
 class ExceptionCenter(Singleton):
     def __init__(self) -> None:
         super().__init__()
-        self._source = Subject()
+        self._source: Subject[BaseException] = Subject()
 
     @property
-    def exceptions(self) -> Observable:
+    def exceptions(self) -> Observable[BaseException]:
         return self._source
 
     def submit(self, exc: BaseException) -> None:

src/blrec/flv/common.py

@@ -1,17 +1,16 @@
 from __future__ import annotations
+
 from typing import Any, Dict, Iterable, Iterator, Mapping, Optional, Union
-from typing_extensions import TypeGuard
 
 import attr
+from typing_extensions import TypeGuard
 
-from .io import FlvReader
-from .io_protocols import RandomIO
 from . import scriptdata
 from .avc import extract_resolution
+from .io import FlvReader
+from .io_protocols import RandomIO
+from .models import AudioTag, AVCPacketType, FlvTag, ScriptTag, TagType, VideoTag
 from .utils import OffsetRepositor
-from .models import (
-    AVCPacketType, FlvTag, AudioTag, ScriptTag, TagType, VideoTag
-)
 
 
 def read_tags(
@@ -41,6 +40,8 @@ def read_tags_in_duration(
         yield tag
         if tag.timestamp > 0:
             break
+    else:
+        raise EOFError('no tags')
 
     start = tag.timestamp
     end = start + duration
@@ -52,14 +53,14 @@ def read_tags_in_duration(
 
 
 def peek_tags(
-    file: RandomIO, reader: FlvReader, count: int, *, no_body: bool = False,
+    file: RandomIO, reader: FlvReader, count: int, *, no_body: bool = False
 ) -> Iterator[FlvTag]:
     with OffsetRepositor(file):
         yield from read_tags(reader, count, no_body=no_body)
 
 
 def rpeek_tags(
-    file: RandomIO, reader: FlvReader, count: int, *, no_body: bool = False,
+    file: RandomIO, reader: FlvReader, count: int, *, no_body: bool = False
 ) -> Iterator[FlvTag]:
     with OffsetRepositor(file):
         yield from rread_tags(reader, count, no_body=no_body)
@@ -72,9 +73,7 @@ def find_metadata_tag(tags: Iterable[FlvTag]) -> Optional[ScriptTag]:
     return None
 
 
-def find_header_tag(
-    tags: Iterable[FlvTag]
-) -> Optional[Union[AudioTag, VideoTag]]:
+def find_header_tag(tags: Iterable[FlvTag]) -> Optional[Union[AudioTag, VideoTag]]:
     for tag in tags:
         if is_sequence_header(tag):
             return tag
@@ -219,9 +218,7 @@ def enrich_metadata(
 
 
 def update_metadata(
-    metadata_tag: ScriptTag,
-    metadata: Mapping[str, Any],
-    offset: Optional[int] = None,
+    metadata_tag: ScriptTag, metadata: Mapping[str, Any], offset: Optional[int] = None
 ) -> ScriptTag:
     original_tag_size = metadata_tag.tag_size
     new_tag = enrich_metadata(metadata_tag, metadata, offset)
@@ -234,7 +231,8 @@ def ensure_order(metadata: Dict[str, Any]) -> Dict[str, Any]:
     # some typical properties such as 'keyframes' must be before some custom
     # properties such as 'Comment' otherwise, it won't take effect in some
     # players!
-    from .data_analyser import MetaData
+    from .operators import MetaData
+
     typical_props = attr.fields_dict(MetaData).keys()
     return {
         **{k: v for k, v in metadata.items() if k in typical_props},

src/blrec/flv/exceptions.py

@@ -1,4 +1,3 @@
-
 class FlvDataError(ValueError):
     ...
 
@@ -17,23 +16,3 @@ class FlvStreamCorruptedError(Exception):
 
 class FlvFileCorruptedError(Exception):
     ...
-
-
-class AudioParametersChanged(Exception):
-    ...
-
-
-class VideoParametersChanged(Exception):
-    ...
-
-
-class FileSizeOverLimit(Exception):
-    ...
-
-
-class DurationOverLimit(Exception):
-    ...
-
-
-class CutStream(Exception):
-    ...

src/blrec/flv/format.py

@@ -37,8 +37,15 @@ __all__ = 'FlvParser', 'FlvDumper'
 
 
 class FlvParser:
-    def __init__(self, stream: RandomIO) -> None:
+    def __init__(
+        self,
+        stream: RandomIO,
+        backup_timestamp: bool = False,
+        restore_timestamp: bool = False,
+    ) -> None:
         self._stream = stream
+        self._backup_timestamp = backup_timestamp
+        self._restore_timestamp = restore_timestamp
         self._reader = StructReader(stream)
 
     def parse_header(self) -> FlvHeader:
@@ -108,19 +115,42 @@ class FlvParser:
 
     def parse_flv_tag_header(self, data: bytes) -> FlvTagHeader:
         reader = StructReader(BytesIO(data))
+
         flag = reader.read_ui8()
         filtered = bool(flag & 0b0010_0000)
         if filtered:
             raise FlvDataError('Unsupported Filtered FLV Tag', data)
+
         tag_type = TagType(flag & 0b0001_1111)
         data_size = reader.read_ui24()
         timestamp = reader.read_ui24()
         timestamp_extended = reader.read_ui8()
-        timestamp = timestamp_extended << 24 | timestamp
         stream_id = reader.read_ui24()
-        return FlvTagHeader(
-            filtered, tag_type, data_size, timestamp, stream_id
-        )
+
+        if self._backup_timestamp:
+            return FlvTagHeader(
+                filtered=filtered,
+                tag_type=tag_type,
+                data_size=data_size,
+                timestamp=timestamp_extended << 24 | timestamp,
+                stream_id=timestamp,
+            )
+        elif self._restore_timestamp:
+            return FlvTagHeader(
+                filtered=filtered,
+                tag_type=tag_type,
+                data_size=data_size,
+                timestamp=stream_id,
+                stream_id=stream_id,
+            )
+        else:
+            return FlvTagHeader(
+                filtered=filtered,
+                tag_type=tag_type,
+                data_size=data_size,
+                timestamp=timestamp_extended << 24 | timestamp,
+                stream_id=stream_id,
+            )
 
     def parse_audio_tag_header(self, data: bytes) -> AudioTagHeader:
         reader = StructReader(BytesIO(data))

src/blrec/flv/helpers.py

@@ -7,7 +7,7 @@ from .common import (
     is_audio_tag, is_metadata_tag, is_script_tag, is_video_tag, read_tags,
     parse_metadata, find_metadata_tag,
 )
-from .stream_processor import JoinPoint
+from .operators import JoinPoint
 from .utils import format_timestamp
 from ..path import extra_metadata_path
 
@@ -32,7 +32,7 @@ def make_comment_for_joinpoints(join_points: Iterable[JoinPoint]) -> str:
         '流中断拼接详情\n' +
         '\n'.join((
             '时间戳：{}, 无缝拼接：{}'.format(
-                format_timestamp(p.timestamp),
+                format_timestamp(int(p.timestamp)),
                 '是' if p.seamless else '否',
             )
             for p in join_points

src/blrec/flv/io.py

@@ -16,9 +16,21 @@ logger = logging.getLogger(__name__)
 
 
 class FlvReader:
-    def __init__(self, stream: RandomIO) -> None:
+    def __init__(
+        self,
+        stream: RandomIO,
+        *,
+        backup_timestamp: bool = False,
+        restore_timestamp: bool = False,
+    ) -> None:
         self._stream = stream
-        self._parser = FlvParser(stream)
+        self._backup_timestamp = backup_timestamp
+        self._restore_timestamp = restore_timestamp
+        self._parser = FlvParser(
+            stream,
+            backup_timestamp=backup_timestamp,
+            restore_timestamp=restore_timestamp,
+        )
 
     def read_header(self) -> FlvHeader:
         header = self._parser.parse_header()

src/blrec/flv/limit_checker.py

@@ -1,84 +0,0 @@
-from __future__ import annotations
-import logging
-from typing import Optional
-
-
-from .models import FlvHeader, FlvTag, BACK_POINTER_SIZE, VideoTag
-from .exceptions import FileSizeOverLimit, DurationOverLimit
-from .common import is_video_nalu_keyframe
-
-
-__all__ = 'LimitChecker',
-
-
-logger = logging.getLogger(__name__)
-
-
-class LimitChecker:
-    def __init__(
-        self,
-        filesize_limit: int = 0,  # file size in bytes, no limit by default.
-        duration_limit: int = 0,  # duration in seconds, no limit by default.
-    ) -> None:
-        self.filesize_limit = filesize_limit
-        self.duration_limit = duration_limit
-        self.reset()
-
-    def is_filesize_over_limit(self) -> bool:
-        return (
-            self._filesize + self._max_size_between_keyframes >=
-            self.filesize_limit
-        )
-
-    def is_duration_over_limit(self) -> bool:
-        return (
-            self._duration + self._max_duration_between_keyframes >=
-            self.duration_limit
-        )
-
-    @property
-    def last_keyframe_tag(self) -> Optional[VideoTag]:
-        return self._last_keyframe_tag
-
-    def reset(self) -> None:
-        self._filesize = 0
-        self._duration = 0.0
-        self._max_size_between_keyframes = 0
-        self._max_duration_between_keyframes = 0.0
-        self._header_checked = False
-        self._last_keyframe_tag: Optional[VideoTag] = None
-
-    def check_header(self, header: FlvHeader) -> None:
-        assert not self._header_checked
-        self._header_checked = True
-        self._filesize += header.size + BACK_POINTER_SIZE
-
-    def check_tag(self, tag: FlvTag) -> None:
-        self._filesize += tag.tag_size + BACK_POINTER_SIZE
-        self._duration = tag.timestamp / 1000
-
-        if not is_video_nalu_keyframe(tag):
-            return
-
-        if self._last_keyframe_tag is not None:
-            self._max_size_between_keyframes = max(
-                self._max_size_between_keyframes,
-                tag.offset - self._last_keyframe_tag.offset,
-            )
-            self._max_duration_between_keyframes = max(
-                self._max_duration_between_keyframes,
-                (tag.timestamp - self._last_keyframe_tag.timestamp) / 1000,
-            )
-        self._last_keyframe_tag = tag
-
-        if self.filesize_limit > 0 and self.is_filesize_over_limit():
-            logger.debug('File size will be over the limit: {} + {}'.format(
-                self._filesize, self._max_size_between_keyframes,
-            ))
-            raise FileSizeOverLimit()
-
-        if self.duration_limit > 0 and self.is_duration_over_limit():
-            logger.debug('Duration will be over the limit: {} + {}'.format(
-                self._duration, self._max_duration_between_keyframes,
-            ))
-            raise DurationOverLimit()

src/blrec/flv/metadata_dumper.py

@@ -0,0 +1,78 @@
+import json
+import logging
+from contextlib import suppress
+from typing import List, Optional
+
+import attr
+
+from ..flv import operators as flv_ops
+from ..path import extra_metadata_path
+from ..utils.mixins import SwitchableMixin
+
+__all__ = ('MetadataDumper',)
+
+logger = logging.getLogger(__name__)
+
+
+class MetadataDumper(SwitchableMixin):
+    def __init__(
+        self,
+        dumper: flv_ops.Dumper,
+        analyser: flv_ops.Analyser,
+        joinpoint_extractor: flv_ops.JoinPointExtractor,
+    ) -> None:
+        super().__init__()
+
+        self._dumper = dumper
+        self._analyser = analyser
+        self._joinpoint_extractor = joinpoint_extractor
+
+        self._last_metadata: Optional[flv_ops.MetaData] = None
+        self._last_join_points: Optional[List[flv_ops.JoinPoint]] = None
+
+    def _do_enable(self) -> None:
+        self._metadata_subscription = self._analyser.metadatas.subscribe(
+            self._update_metadata
+        )
+        self._join_points_subscription = (
+            self._joinpoint_extractor.join_points.subscribe(self._update_join_points)
+        )
+        self._file_closed_subscription = self._dumper.file_closed.subscribe(
+            self._dump_metadata
+        )
+        logger.debug('Enabled metadata dumper')
+
+    def _do_disable(self) -> None:
+        with suppress(Exception):
+            self._metadata_subscription.dispose()
+        with suppress(Exception):
+            self._join_points_subscription.dispose()
+        with suppress(Exception):
+            self._file_closed_subscription.dispose()
+        logger.debug('Disabled metadata dumper')
+
+    def _update_metadata(self, metadata: flv_ops.MetaData) -> None:
+        self._last_metadata = metadata
+
+    def _update_join_points(self, join_points: List[flv_ops.JoinPoint]) -> None:
+        self._last_join_points = join_points
+
+    def _dump_metadata(self, video_path: str) -> None:
+        path = extra_metadata_path(video_path)
+        logger.debug(f"Dumping metadata to file: '{path}'")
+
+        assert self._last_metadata is not None
+        assert self._last_join_points is not None
+
+        data = attr.asdict(self._last_metadata, filter=lambda a, v: v is not None)
+        data['joinpoints'] = list(
+            map(lambda p: p.to_metadata_value(), self._last_join_points)
+        )
+
+        try:
+            with open(path, 'wt', encoding='utf8') as file:
+                json.dump(data, file)
+        except Exception as e:
+            logger.error(f'Failed to dump metadata: {e}')
+        else:
+            logger.debug(f"Successfully dumped metadata to file: '{path}'")

src/blrec/flv/metadata_injection.py

@@ -0,0 +1,60 @@
+import logging
+import os
+from datetime import datetime
+from typing import Any, Dict
+
+import attr
+from reactivex import Observable
+from reactivex import operators as ops
+
+from ..utils import operators as utils_ops
+from . import operators as flv_ops
+from .helpers import make_comment_for_joinpoints
+from .operators import JoinPoint
+from .operators.helpers import from_file
+
+__all__ = 'InjectingProgress', 'inject_metadata'
+
+
+logger = logging.getLogger(__name__)
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True)
+class InjectingProgress:
+    count: int
+    total: int
+
+
+def inject_metadata(
+    path: str, metadata: Dict[str, Any], *, show_progress: bool = False
+) -> Observable[InjectingProgress]:
+    filesize = os.path.getsize(path)
+    append_comment_for_joinpoints(metadata)
+
+    root, ext = os.path.splitext(path)
+    temp_path = f'{root}_injecting{ext}'
+    filename = os.path.basename(path)
+
+    return from_file(path).pipe(
+        flv_ops.Injector(lambda: metadata),
+        flv_ops.Dumper(lambda: (temp_path, int(datetime.now().timestamp()))),
+        flv_ops.ProgressBar(
+            desc='Injecting',
+            postfix=filename,
+            total=filesize,
+            disable=not show_progress,
+        ),
+        ops.map(lambda i: len(i)),
+        ops.scan(lambda acc, x: acc + x, 0),
+        ops.map(lambda s: InjectingProgress(s, filesize)),
+        utils_ops.replace(temp_path, path),
+    )
+
+
+def append_comment_for_joinpoints(metadata: Dict[str, Any]) -> None:
+    if join_points := metadata.get('joinpoints'):
+        join_points = map(JoinPoint.from_metadata_value, join_points)
+        if 'Comment' in metadata:
+            metadata['Comment'] += '\n\n' + make_comment_for_joinpoints(join_points)
+        else:
+            metadata['Comment'] = make_comment_for_joinpoints(join_points)

src/blrec/flv/metadata_injector.py

@@ -1,148 +0,0 @@
-import os
-import logging
-from typing import Any, Dict, Optional
-
-import attr
-from rx import create, operators as op
-from rx.subject import Subject
-from rx.core import Observable
-from rx.core.typing import Observer, Scheduler, Disposable
-from rx.scheduler.currentthreadscheduler import CurrentThreadScheduler
-from tqdm import tqdm
-
-from .stream_processor import StreamProcessor, BaseOutputFileManager, JoinPoint
-from .helpers import get_metadata, make_comment_for_joinpoints
-from ..logging.room_id import with_room_id
-
-
-__all__ = 'MetadataInjector', 'InjectProgress', 'inject_metadata'
-
-
-logger = logging.getLogger(__name__)
-
-
-@attr.s(auto_attribs=True, slots=True, frozen=True)
-class InjectProgress:
-    time: int
-    duration: int
-
-
-class MetadataInjector:
-    def __init__(self, in_path: str, out_path: str) -> None:
-        self._in_path = in_path
-        self._file_manager = OutputFileManager(out_path)
-        self._duration: int = 0
-        self._progress_updates = Subject()
-
-    @property
-    def progress_updates(self) -> Observable:
-        return self._progress_updates
-
-    def inject(self, in_metadata: Dict[str, Any]) -> None:
-        metadata = get_metadata(self._in_path)
-        metadata.update(in_metadata)
-
-        self._duration = int(float(metadata['duration']) * 1000)
-        self._progress_updates.on_next(InjectProgress(0, self._duration))
-
-        self._append_comment_for_joinpoints(metadata)
-
-        processor = StreamProcessor(
-            self._file_manager,
-            metadata=metadata,
-            disable_limit=True,
-        )
-
-        def update_progress(time: int) -> None:
-            progress = InjectProgress(time, self._duration)
-            self._progress_updates.on_next(progress)
-
-        processor.time_updates.subscribe(update_progress)
-
-        with open(self._in_path, 'rb') as in_file:
-            processor.process_stream(in_file)
-            processor.finalize()
-
-        progress = InjectProgress(self._duration, self._duration)
-        self._progress_updates.on_next(progress)
-
-    @staticmethod
-    def _append_comment_for_joinpoints(metadata: Dict[str, Any]) -> None:
-        if (join_points := metadata.get('joinpoints')):
-            join_points = map(JoinPoint.from_metadata_value, join_points)
-            if 'Comment' in metadata:
-                metadata['Comment'] += '\n\n' + \
-                    make_comment_for_joinpoints(join_points)
-            else:
-                metadata['Comment'] = make_comment_for_joinpoints(join_points)
-
-
-class OutputFileManager(BaseOutputFileManager):
-    def __init__(self, out_path: str) -> None:
-        super().__init__()
-        self._out_path = out_path
-
-    def _make_path(self) -> str:
-        return self._out_path
-
-
-def inject_metadata(
-    path: str,
-    metadata: Dict[str, Any],
-    *,
-    report_progress: bool = False,
-    room_id: Optional[int] = None,
-) -> Observable:
-    def subscribe(
-        observer: Observer[InjectProgress],
-        scheduler: Optional[Scheduler] = None,
-    ) -> Disposable:
-        _scheduler = scheduler or CurrentThreadScheduler.singleton()
-
-        def action(scheduler, state):  # type: ignore
-            root, ext = os.path.splitext(path)
-            out_path = f'{root}_inject_metadata{ext}'
-            injector = MetadataInjector(path, out_path)
-            file_name = os.path.basename(path)
-
-            with tqdm(desc='Injecting', unit='ms', postfix=file_name) as pbar:
-                def reset(progress: InjectProgress) -> None:
-                    pbar.reset(progress.duration)
-
-                def update(progress: InjectProgress) -> None:
-                    pbar.update(progress.time - pbar.n)
-
-                injector.progress_updates.pipe(op.first()).subscribe(reset)
-                injector.progress_updates.pipe(op.skip(1)).subscribe(update)
-
-                if report_progress:
-                    injector.progress_updates.subscribe(
-                        lambda p: observer.on_next(p)
-                    )
-
-                try:
-                    logger.info(f"Injecting metadata for '{path}' ...")
-                    injector.inject(metadata)
-                except Exception as e:
-                    logger.error(
-                        f"Failed to inject metadata for '{path}': {repr(e)}"
-                    )
-                    observer.on_error(e)
-                else:
-                    logger.info(f"Successfully inject metadata for '{path}'")
-                    try:
-                        os.replace(out_path, path)
-                    except Exception as e:
-                        logger.error(
-                            f"Failed to replace file {path} with '{out_path}'"
-                        )
-                        observer.on_error(e)
-                    else:
-                        observer.on_completed()
-
-        if room_id is not None:
-            return _scheduler.schedule(with_room_id(room_id)(action))
-        else:
-            return _scheduler.schedule(action)
-
-    return create(subscribe)

src/blrec/flv/models.py

@@ -106,6 +106,9 @@ class FlvHeader:
         else:
             self.type_flag &= ~0b0000_0100
 
+    def __len__(self) -> int:
+        return self.size
+
     @property
     def size(self) -> int:
         return self.data_offset
@@ -116,8 +119,8 @@ class FlvTagHeader:
     filtered: bool
     tag_type: TagType
     data_size: int = attr.ib(validator=[non_negative_integer_validator])
-    timestamp: int = attr.ib(validator=[non_negative_integer_validator])
-    stream_id: int = attr.ib(validator=[non_negative_integer_validator])
+    timestamp: int
+    stream_id: int
 
 
 @attr.s(auto_attribs=True, slots=True, frozen=True)
@@ -151,6 +154,9 @@ class FlvTag(ABC, FlvTagHeader):
     offset: int = attr.ib(validator=[non_negative_integer_validator])
     body: Optional[bytes] = attr.ib(default=None, repr=cksum)
 
+    def __len__(self) -> int:
+        return self.tag_size
+
     @property
     @abstractmethod
     def header_size(self) -> int:

src/blrec/flv/operators/__init__.py

@@ -0,0 +1,38 @@
+from .analyse import Analyser, KeyFrames, MetaData
+from .concat import JoinPoint, JoinPointData, JoinPointExtractor, concat
+from .correct import correct
+from .cut import Cutter
+from .defragment import defragment
+from .dump import Dumper
+from .fix import fix
+from .inject import Injector
+from .limit import Limiter
+from .parse import parse
+from .probe import Prober, StreamProfile
+from .process import process
+from .progress import ProgressBar
+from .split import split
+
+__all__ = (
+    'Analyser',
+    'concat',
+    'concat',
+    'correct',
+    'Cutter',
+    'defragment',
+    'Dumper',
+    'fix',
+    'Injector',
+    'JoinPoint',
+    'JoinPointData',
+    'JoinPointExtractor',
+    'KeyFrames',
+    'Limiter',
+    'MetaData',
+    'parse',
+    'Prober',
+    'process',
+    'ProgressBar',
+    'split',
+    'StreamProfile',
+)

src/blrec/flv/operators/analyse.py

@@ -1,237 +1,331 @@
-"""
-Analyse FLV file to make MetaData
-ref: https://github.com/ioppermann/yamdi/blob/master/yamdi.c
-"""
-from __future__ import annotations
-from typing import List, Optional
-
-import attr
-
-from .models import (
-    AudioTag, FlvHeader, FlvTag, SoundType, VideoTag, ScriptTag,
-    BACK_POINTER_SIZE
-)
-from .common import Resolution, is_audio_tag, is_script_tag, is_video_tag
-
-
-__all__ = 'DataAnalyser', 'MetaData', 'KeyFrames'
-
-
-@attr.s(auto_attribs=True, slots=True, frozen=True, kw_only=True)
-class KeyFrames:
-    times: List[float]
-    filepositions: List[float]
-
-
-@attr.s(auto_attribs=True, slots=True, frozen=True, kw_only=True)
-class MetaData:
-    hasAudio: bool
-    hasVideo: bool
-    hasMetadata: bool
-    hasKeyframes: bool
-    canSeekToEnd: bool
-    duration: float
-    datasize: float
-    filesize: float
-
-    audiosize: Optional[float] = None
-    audiocodecid: Optional[float] = None
-    audiodatarate: Optional[float] = None
-    audiosamplerate: Optional[float] = None
-    audiosamplesize: Optional[float] = None
-    stereo: Optional[bool] = None
-
-    videosize: float
-    framerate: float
-    videocodecid: float
-    videodatarate: float
-    width: float
-    height: float
-
-    lasttimestamp: float
-    lastkeyframelocation: float
-    lastkeyframetimestamp: float
-    keyframes: KeyFrames
-
-
-class DataAnalyser:
-    def __init__(self) -> None:
-        self.reset()
-
-    def reset(self) -> None:
-        self._num_of_tags = 0
-        self._num_of_audio_tags = 0
-        self._num_of_video_tags = 0
-        self._size_of_tags = 0
-        self._size_of_audio_tags = 0
-        self._size_of_video_tags = 0
-        self._size_of_data = 0
-        self._size_of_audio_data = 0
-        self._size_of_video_data = 0
-        self._last_timestamp = 0
-        self._last_timestamp_of_audio = 0
-        self._last_timestamp_of_video = 0
-        self._keyframe_timestamps: List[int] = []
-        self._keyframe_filepositions: List[int] = []
-        self._resolution: Optional[Resolution] = None
-
-        self._has_audio = False
-        self._has_video = False
-        self._header_analysed = False
-        self._audio_analysed = False
-        self._video_analysed = False
-
-    @property
-    def last_timestamp(self) -> int:
-        return self._last_timestamp
-
-    def analyse_header(self, header: FlvHeader) -> None:
-        assert not self._header_analysed
-        self._header_analysed = True
-        self._size_of_flv_header = header.size
-
-    def analyse_tag(self, tag: FlvTag) -> None:
-        if is_audio_tag(tag):
-            self._analyse_audio_tag(tag)
-        elif is_video_tag(tag):
-            self._analyse_video_tag(tag)
-        elif is_script_tag(tag):
-            self._analyse_script_tag(tag)
-        else:
-            raise ValueError('Invalid tag type')
-
-        self._num_of_tags += 1
-        self._size_of_tags += tag.tag_size
-        self._size_of_data += tag.data_size
-        self._last_timestamp = tag.timestamp
-
-    def get_real_resolution(self) -> Optional[Resolution]:
-        return self._resolution
-
-    def calc_frame_rate(self) -> float:
-        try:
-            return (
-                self._num_of_video_tags / self._last_timestamp_of_video * 1000
-            )
-        except ZeroDivisionError:
-            return 0.0
-
-    def calc_audio_data_rate(self) -> float:
-        try:
-            return self._size_of_audio_data * 8 / self._last_timestamp_of_audio
-        except ZeroDivisionError:
-            return 0.0
-
-    def calc_video_data_rate(self) -> float:
-        try:
-            return self._size_of_video_data * 8 / self._last_timestamp_of_video
-        except ZeroDivisionError:
-            return 0.0
-
-    def calc_data_size(self) -> int:
-        return (
-            self._size_of_audio_tags +
-            self._num_of_audio_tags * BACK_POINTER_SIZE +
-            self._size_of_video_tags +
-            self._num_of_video_tags * BACK_POINTER_SIZE
-        )
-
-    def calc_file_size(self) -> int:
-        return (
-            self._size_of_flv_header + BACK_POINTER_SIZE +
-            self._size_of_tags + self._num_of_tags * BACK_POINTER_SIZE
-        )
-
-    def make_keyframes(self) -> KeyFrames:
-        return KeyFrames(
-            times=list(map(lambda t: t / 1000, self._keyframe_timestamps)),
-            filepositions=list(map(float, self._keyframe_filepositions)),
-        )
-
-    def make_metadata(self) -> MetaData:
-        assert self._header_analysed
-        assert self._has_audio == self._audio_analysed
-        assert self._has_video and self._video_analysed
-        assert self._resolution is not None
-
-        if not self._has_audio:
-            audiosize = None
-            audiocodecid = None
-            audiodatarate = None
-            audiosamplerate = None
-            audiosamplesize = None
-            stereo = None
-        else:
-            audiosize = float(self._size_of_audio_tags)
-            audiocodecid = float(self._audio_codec_id)
-            audiodatarate = self.calc_audio_data_rate()
-            audiosamplerate = float(self._audio_sample_rate)
-            audiosamplesize = float(self._audio_sample_size)
-            stereo = self._stereo
-
-        keyframes = self.make_keyframes()
-
-        return MetaData(
-            hasAudio=self._has_audio,
-            hasVideo=self._has_video,
-            hasMetadata=True,
-            hasKeyframes=len(self._keyframe_timestamps) != 0,
-            canSeekToEnd=(
-                self._last_timestamp_of_video == self._keyframe_timestamps[-1]
-            ),
-            duration=self._last_timestamp / 1000,
-            datasize=float(self.calc_data_size()),
-            filesize=float(self.calc_file_size()),
-            audiosize=audiosize,
-            audiocodecid=audiocodecid,
-            audiodatarate=audiodatarate,
-            audiosamplerate=audiosamplerate,
-            audiosamplesize=audiosamplesize,
-            stereo=stereo,
-            videosize=float(self._size_of_video_tags),
-            framerate=self.calc_frame_rate(),
-            videocodecid=float(self._video_codec_id),
-            videodatarate=self.calc_video_data_rate(),
-            width=float(self._resolution.width),
-            height=float(self._resolution.height),
-            lasttimestamp=self._last_timestamp / 1000,
-            lastkeyframelocation=keyframes.filepositions[-1],
-            lastkeyframetimestamp=keyframes.times[-1],
-            keyframes=keyframes,
-        )
-
-    def _analyse_audio_tag(self, tag: AudioTag) -> None:
-        if not self._audio_analysed:
-            self._has_audio = True
-            self._audio_analysed = True
-            self._audio_codec_id = tag.sound_format.value
-            self._audio_sample_rate = tag.sound_rate.value
-            self._audio_sample_size = tag.sound_size.value
-            self._stereo = tag.sound_type == SoundType.STEREO
-
-        self._num_of_audio_tags += 1
-        self._size_of_audio_tags += tag.tag_size
-        self._size_of_audio_data += tag.data_size
-        self._last_timestamp_of_audio = tag.timestamp
-
-    def _analyse_video_tag(self, tag: VideoTag) -> None:
-        if tag.is_keyframe():
-            self._keyframe_timestamps.append(tag.timestamp)
-            self._keyframe_filepositions.append(tag.offset)
-            if tag.is_avc_header():
-                self._resolution = Resolution.from_aac_sequence_header(tag)
-        else:
-            pass
-
-        if not self._video_analysed:
-            self._has_video = True
-            self._video_analysed = True
-            self._video_codec_id = tag.codec_id.value
-
-        self._num_of_video_tags += 1
-        self._size_of_video_tags += tag.tag_size
-        self._size_of_video_data += tag.data_size
-        self._last_timestamp_of_video = tag.timestamp
-
-    def _analyse_script_tag(self, tag: ScriptTag) -> None:
-        pass
+"""
+Analyse the FLV stream to make MetaData
+ref: https://github.com/ioppermann/yamdi/blob/master/yamdi.c
+"""
+from __future__ import annotations
+
+import logging
+from typing import List, Optional, TypedDict
+
+import attr
+from reactivex import Observable, Subject, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+
+from ..common import Resolution, is_audio_tag, is_script_tag, is_video_tag
+from ..models import (
+    BACK_POINTER_SIZE,
+    AudioTag,
+    FlvHeader,
+    FlvTag,
+    ScriptTag,
+    SoundType,
+    VideoTag,
+)
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = 'Analyser', 'MetaData', 'KeyFrames'
+
+logger = logging.getLogger(__name__)
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True, kw_only=True)
+class KeyFrames:
+    times: List[float]
+    filepositions: List[float]
+
+
+class KeyFramesDict(TypedDict):
+    times: List[float]
+    filepositions: List[float]
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True, kw_only=True)
+class MetaData:
+    hasAudio: bool
+    hasVideo: bool
+    hasMetadata: bool
+    hasKeyframes: bool
+    canSeekToEnd: bool
+    duration: float
+    datasize: float
+    filesize: float
+
+    audiosize: Optional[float] = None
+    audiocodecid: Optional[float] = None
+    audiodatarate: Optional[float] = None
+    audiosamplerate: Optional[float] = None
+    audiosamplesize: Optional[float] = None
+    stereo: Optional[bool] = None
+
+    videosize: float
+    framerate: float
+    videocodecid: float
+    videodatarate: float
+    width: float
+    height: float
+
+    lasttimestamp: float
+    lastkeyframelocation: float
+    lastkeyframetimestamp: float
+    keyframes: KeyFrames
+
+
+class MetaDataDict:
+    hasAudio: bool
+    hasVideo: bool
+    hasMetadata: bool
+    hasKeyframes: bool
+    canSeekToEnd: bool
+    duration: float
+    datasize: float
+    filesize: float
+
+    audiosize: Optional[float] = None
+    audiocodecid: Optional[float] = None
+    audiodatarate: Optional[float] = None
+    audiosamplerate: Optional[float] = None
+    audiosamplesize: Optional[float] = None
+    stereo: Optional[bool] = None
+
+    videosize: float
+    framerate: float
+    videocodecid: float
+    videodatarate: float
+    width: float
+    height: float
+
+    lasttimestamp: float
+    lastkeyframelocation: float
+    lastkeyframetimestamp: float
+    keyframes: KeyFramesDict
+
+
+class Analyser:
+    def __init__(self) -> None:
+        self._metadatas: Subject[MetaData] = Subject()
+        self._reset()
+
+    def _reset(self) -> None:
+        self._num_of_tags = 0
+        self._num_of_audio_tags = 0
+        self._num_of_video_tags = 0
+        self._size_of_tags = 0
+        self._size_of_audio_tags = 0
+        self._size_of_video_tags = 0
+        self._size_of_data = 0
+        self._size_of_audio_data = 0
+        self._size_of_video_data = 0
+        self._last_timestamp = 0
+        self._last_timestamp_of_audio = 0
+        self._last_timestamp_of_video = 0
+        self._keyframe_timestamps: List[int] = []
+        self._keyframe_filepositions: List[int] = []
+        self._resolution: Optional[Resolution] = None
+
+        self._has_audio = False
+        self._has_video = False
+        self._audio_analysed = False
+        self._video_analysed = False
+
+    @property
+    def metadatas(self) -> Observable[MetaData]:
+        return self._metadatas
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._analyse(source)
+
+    def get_real_resolution(self) -> Optional[Resolution]:
+        return self._resolution
+
+    def calc_frame_rate(self) -> float:
+        try:
+            return self._num_of_video_tags / self._last_timestamp_of_video * 1000
+        except ZeroDivisionError:
+            return 0.0
+
+    def calc_audio_data_rate(self) -> float:
+        try:
+            return self._size_of_audio_data * 8 / self._last_timestamp_of_audio
+        except ZeroDivisionError:
+            return 0.0
+
+    def calc_video_data_rate(self) -> float:
+        try:
+            return self._size_of_video_data * 8 / self._last_timestamp_of_video
+        except ZeroDivisionError:
+            return 0.0
+
+    def calc_data_size(self) -> int:
+        return (
+            self._size_of_audio_tags
+            + self._num_of_audio_tags * BACK_POINTER_SIZE
+            + self._size_of_video_tags
+            + self._num_of_video_tags * BACK_POINTER_SIZE
+        )
+
+    def calc_file_size(self) -> int:
+        return (
+            self._size_of_flv_header
+            + BACK_POINTER_SIZE
+            + self._size_of_tags
+            + self._num_of_tags * BACK_POINTER_SIZE
+        )
+
+    def make_keyframes(self) -> KeyFrames:
+        return KeyFrames(
+            times=list(map(lambda t: t / 1000, self._keyframe_timestamps)),
+            filepositions=list(map(float, self._keyframe_filepositions)),
+        )
+
+    def make_metadata(self) -> MetaData:
+        assert self._has_audio == self._audio_analysed
+        assert self._has_video and self._video_analysed
+        assert self._resolution is not None
+
+        if not self._has_audio:
+            audiosize = None
+            audiocodecid = None
+            audiodatarate = None
+            audiosamplerate = None
+            audiosamplesize = None
+            stereo = None
+        else:
+            audiosize = float(self._size_of_audio_tags)
+            audiocodecid = float(self._audio_codec_id)
+            audiodatarate = self.calc_audio_data_rate()
+            audiosamplerate = float(self._audio_sample_rate)
+            audiosamplesize = float(self._audio_sample_size)
+            stereo = self._stereo
+
+        keyframes = self.make_keyframes()
+
+        return MetaData(
+            hasAudio=self._has_audio,
+            hasVideo=self._has_video,
+            hasMetadata=True,
+            hasKeyframes=len(self._keyframe_timestamps) != 0,
+            canSeekToEnd=(
+                self._last_timestamp_of_video == self._keyframe_timestamps[-1]
+            ),
+            duration=self._last_timestamp / 1000,
+            datasize=float(self.calc_data_size()),
+            filesize=float(self.calc_file_size()),
+            audiosize=audiosize,
+            audiocodecid=audiocodecid,
+            audiodatarate=audiodatarate,
+            audiosamplerate=audiosamplerate,
+            audiosamplesize=audiosamplesize,
+            stereo=stereo,
+            videosize=float(self._size_of_video_tags),
+            framerate=self.calc_frame_rate(),
+            videocodecid=float(self._video_codec_id),
+            videodatarate=self.calc_video_data_rate(),
+            width=float(self._resolution.width),
+            height=float(self._resolution.height),
+            lasttimestamp=self._last_timestamp / 1000,
+            lastkeyframelocation=keyframes.filepositions[-1],
+            lastkeyframetimestamp=keyframes.times[-1],
+            keyframes=keyframes,
+        )
+
+    def _analyse(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            stream_index: int = -1
+            subscription = SerialDisposable()
+
+            def push_metadata() -> None:
+                try:
+                    metadata = self.make_metadata()
+                except Exception as e:
+                    logger.warning(f'Failed to make metadata: {repr(e)}')
+                    pass
+                else:
+                    self._metadatas.on_next(metadata)
+
+            def on_next(item: FLVStreamItem) -> None:
+                nonlocal stream_index
+                if isinstance(item, FlvHeader):
+                    stream_index += 1
+                    if stream_index > 0:
+                        push_metadata()
+                    self._reset()
+                    self._analyse_flv_header(item)
+                else:
+                    self._analyse_tag(item)
+                observer.on_next(item)
+
+            def on_completed() -> None:
+                push_metadata()
+                observer.on_completed()
+
+            def on_error(e: Exception) -> None:
+                push_metadata()
+                observer.on_error(e)
+
+            def dispose() -> None:
+                push_metadata()
+
+            subscription.disposable = source.subscribe(
+                on_next, on_error, on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)
+
+    def _analyse_flv_header(self, header: FlvHeader) -> None:
+        self._size_of_flv_header = header.size
+
+    def _analyse_tag(self, tag: FlvTag) -> None:
+        if is_audio_tag(tag):
+            self._analyse_audio_tag(tag)
+        elif is_video_tag(tag):
+            self._analyse_video_tag(tag)
+        elif is_script_tag(tag):
+            self._analyse_script_tag(tag)
+        else:
+            logger.warning(f'Invalid tag type: {tag}')
+
+        self._num_of_tags += 1
+        self._size_of_tags += tag.tag_size
+        self._size_of_data += tag.data_size
+        self._last_timestamp = tag.timestamp
+
+    def _analyse_audio_tag(self, tag: AudioTag) -> None:
+        if not self._audio_analysed:
+            self._has_audio = True
+            self._audio_analysed = True
+            self._audio_codec_id = tag.sound_format.value
+            self._audio_sample_rate = tag.sound_rate.value
+            self._audio_sample_size = tag.sound_size.value
+            self._stereo = tag.sound_type == SoundType.STEREO
+
+        self._num_of_audio_tags += 1
+        self._size_of_audio_tags += tag.tag_size
+        self._size_of_audio_data += tag.data_size
+        self._last_timestamp_of_audio = tag.timestamp
+
+    def _analyse_video_tag(self, tag: VideoTag) -> None:
+        if tag.is_keyframe():
+            self._keyframe_timestamps.append(tag.timestamp)
+            self._keyframe_filepositions.append(self.calc_file_size())
+            if tag.is_avc_header():
+                self._resolution = Resolution.from_aac_sequence_header(tag)
+        else:
+            pass
+
+        if not self._video_analysed:
+            self._has_video = True
+            self._video_analysed = True
+            self._video_codec_id = tag.codec_id.value
+
+        self._num_of_video_tags += 1
+        self._size_of_video_tags += tag.tag_size
+        self._size_of_video_data += tag.data_size
+        self._last_timestamp_of_video = tag.timestamp
+
+    def _analyse_script_tag(self, tag: ScriptTag) -> None:
+        pass

src/blrec/flv/operators/concat.py

@@ -0,0 +1,406 @@
+from __future__ import annotations
+
+import logging
+from enum import IntEnum, auto
+from typing import Callable, List, Optional, TypedDict, cast
+
+import attr
+from reactivex import Observable, Subject, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+from typing_extensions import TypeGuard
+
+from ...utils.hash import cksum
+from ..common import (
+    create_script_tag,
+    is_audio_sequence_header,
+    is_metadata_tag,
+    is_script_tag,
+    is_sequence_header,
+    is_video_sequence_header,
+    parse_scriptdata,
+)
+from ..models import AudioTag, FlvHeader, FlvTag, ScriptTag, VideoTag
+from ..scriptdata import ScriptData
+from ..utils import format_timestamp
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('concat', 'JoinPointExtractor', 'JoinPoint', 'JoinPointData')
+
+logger = logging.getLogger(__name__)
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True)
+class JoinPoint:
+    seamless: bool
+    timestamp: float  # timestamp of previous tag in milliseconds
+    crc32: str  # crc32 of the previous tag
+
+    @classmethod
+    def from_metadata_value(cls, value: JoinPointData) -> JoinPoint:
+        return cls(
+            timestamp=int(value['timestamp']),
+            seamless=value['seamless'],
+            crc32=value['crc32'],
+        )
+
+    def to_metadata_value(self) -> JoinPointData:
+        return dict(
+            timestamp=float(self.timestamp), seamless=self.seamless, crc32=self.crc32
+        )
+
+    def __str__(self) -> str:
+        return 'seamless: {}, timestamp: {}, crc32: {}'.format(
+            'yes' if self.seamless else 'no',
+            format_timestamp(int(self.timestamp)),
+            self.crc32,
+        )
+
+
+class JoinPointData(TypedDict):
+    seamless: bool
+    timestamp: float
+    crc32: str
+
+
+class ACTION(IntEnum):
+    NOOP = auto()
+    CORRECT = auto()
+    GATHER = auto()
+    CANCEL = auto()
+    CONCAT = auto()
+    CONCAT_AND_GATHER = auto()
+
+
+def concat(
+    num_of_last_tags: int = 3, max_duration: int = 20_000
+) -> Callable[[FLVStream], FLVStream]:
+    """Concat FLV streams.
+
+    num_of_last_tags: Number of tags for determining whether or not tags are duplicated
+    max_duration: Max duration in milliseconds the duplicated tags might last
+    """
+
+    def _concat(source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            delta: int = 0
+            action: ACTION = ACTION.NOOP
+            last_tags: List[FlvTag] = []
+            gathered_tags: List[FlvTag] = []
+            last_flv_header: Optional[FlvHeader] = None
+            last_audio_sequence_header: Optional[AudioTag] = None
+            last_video_sequence_header: Optional[VideoTag] = None
+
+            def update_last_tags(tag: FlvTag) -> None:
+                nonlocal last_audio_sequence_header, last_video_sequence_header
+                last_tags.append(tag)
+                if len(last_tags) > num_of_last_tags:
+                    last_tags.pop(0)
+                if is_audio_sequence_header(tag):
+                    last_audio_sequence_header = tag
+                elif is_video_sequence_header(tag):
+                    last_video_sequence_header = tag
+
+            def gather_tags(tag: FlvTag) -> None:
+                nonlocal gathered_tags
+                nonlocal action
+                nonlocal last_audio_sequence_header, last_video_sequence_header
+                if is_audio_sequence_header(tag):
+                    if last_audio_sequence_header is None:
+                        logger.debug(
+                            'Cancel concat due to no last audio sequence header'
+                        )
+                        action = ACTION.CANCEL
+                    else:
+                        if not tag.is_the_same_as(last_audio_sequence_header):
+                            action = ACTION.CANCEL
+                            logger.debug(
+                                'Cancel concat due to audio sequence header changed'
+                            )
+                    last_audio_sequence_header = tag
+                elif is_video_sequence_header(tag):
+                    if last_video_sequence_header is None:
+                        logger.debug(
+                            'Cancel concat due to no last video sequence header'
+                        )
+                        action = ACTION.CANCEL
+                    else:
+                        if not tag.is_the_same_as(last_video_sequence_header):
+                            action = ACTION.CANCEL
+                            logger.debug(
+                                'Cancel concat due to video sequence header changed'
+                            )
+                    last_video_sequence_header = tag
+                gathered_tags.append(tag)
+
+            def has_gathering_completed() -> bool:
+                # XXX: timestamp MUST start from 0 and continuous!
+                # put the correct and fix operator on upstream of this operator
+                # to ensure timestamp start from 0 and continuous!
+                return gathered_tags[-1].timestamp >= max_duration
+
+            def find_last_duplicated_tag(tags: List[FlvTag]) -> int:
+                logger.debug('Finding duplicated tags...')
+
+                last_out_tag = last_tags[-1]
+                logger.debug(f'The last output tag is {last_out_tag}')
+
+                for idx, tag in enumerate(tags):
+                    if not tag.is_the_same_as(last_out_tag):
+                        continue
+
+                    if not all(
+                        map(
+                            lambda t: t[0].is_the_same_as(t[1]),
+                            zip(
+                                tags[max(0, idx - (len(last_tags) - 1)) : idx],
+                                last_tags[:-1],
+                            ),
+                        )
+                    ):
+                        continue
+
+                    logger.debug(f'The last duplicated tag found at {idx} is {tag}')
+                    return idx
+
+                logger.debug('No duplicated tag found')
+                return -1
+
+            def update_delta_duplicated(last_duplicated_tag: FlvTag) -> None:
+                nonlocal delta
+                delta = last_tags[-1].timestamp - last_duplicated_tag.timestamp
+
+            def update_delta_no_duplicated(first_data_tag: FlvTag) -> None:
+                nonlocal delta
+                delta = last_tags[-1].timestamp - first_data_tag.timestamp + 10
+
+            def correct_ts(tag: FlvTag) -> FlvTag:
+                if delta == 0:
+                    return tag
+                return tag.evolve(timestamp=tag.timestamp + delta)
+
+            def make_join_point_tag(tag: FlvTag, seamless: bool) -> ScriptTag:
+                assert tag.body is not None
+                join_point = JoinPoint(
+                    seamless=seamless,
+                    timestamp=float(tag.timestamp),
+                    crc32=cksum(tag.body),
+                )
+                script_data = ScriptData(
+                    name='onJoinPoint', value=attr.asdict(join_point)
+                )
+                script_tag = create_script_tag(script_data)
+                return script_tag
+
+            def do_concat() -> None:
+                logger.debug(
+                    'Concatenating... gathered {} tags, total size: {}'.format(
+                        len(gathered_tags), sum(t.tag_size for t in gathered_tags)
+                    )
+                )
+
+                tags = list(
+                    filter(
+                        lambda tag: not is_metadata_tag(tag)
+                        and not is_sequence_header(tag),
+                        gathered_tags,
+                    )
+                )
+                logger.debug(
+                    '{} data tags, total size: {}'.format(
+                        len(tags), sum(t.tag_size for t in tags)
+                    )
+                )
+
+                if not tags:
+                    return
+
+                if (index := find_last_duplicated_tag(tags)) >= 0:
+                    seamless = True
+                    update_delta_duplicated(tags[index])
+                    logger.debug(f'Updated delta: {delta}, seamless: {seamless}')
+                    tags = tags[index + 1 :]
+                else:
+                    seamless = False
+                    update_delta_no_duplicated(tags[0])
+                    logger.debug(f'Updated delta: {delta}, seamless: {seamless}')
+
+                join_point_tag = make_join_point_tag(last_tags[-1], seamless)
+                observer.on_next(join_point_tag)
+
+                for tag in tags:
+                    tag = correct_ts(tag)
+                    update_last_tags(tag)
+                    observer.on_next(tag)
+                gathered_tags.clear()
+
+            def do_cancel() -> None:
+                logger.debug(
+                    'Cancelling... gathered {} tags, total size: {}'.format(
+                        len(gathered_tags), sum(t.tag_size for t in gathered_tags)
+                    )
+                )
+                assert last_flv_header is not None
+                observer.on_next(last_flv_header)
+                for tag in gathered_tags:
+                    update_last_tags(tag)
+                    observer.on_next(tag)
+                gathered_tags.clear()
+
+            def on_next(item: FLVStreamItem) -> None:
+                nonlocal action
+                nonlocal last_flv_header
+
+                if isinstance(item, FlvHeader):
+                    if last_flv_header is None:
+                        logger.debug('No operation needed for the first stream')
+                        last_flv_header = item
+                        action = ACTION.NOOP
+                        observer.on_next(item)
+                    else:
+                        logger.debug('Gathering tags for deduplication...')
+                        last_flv_header = item
+                        if action == ACTION.GATHER:
+                            action = ACTION.CONCAT_AND_GATHER
+                        else:
+                            action = ACTION.GATHER
+                    return
+
+                tag = item
+
+                while True:
+                    if action == ACTION.NOOP:
+                        update_last_tags(tag)
+                        observer.on_next(tag)
+                        return
+
+                    if action == ACTION.CORRECT:
+                        tag = correct_ts(tag)
+                        update_last_tags(tag)
+                        observer.on_next(tag)
+                        return
+
+                    if action in (ACTION.CONCAT, ACTION.CONCAT_AND_GATHER):
+                        do_concat()
+                        if action == ACTION.CONCAT_AND_GATHER:
+                            action = ACTION.GATHER
+                        else:
+                            action = ACTION.CORRECT
+                            return
+
+                    if action == ACTION.GATHER:
+                        gather_tags(tag)
+                        if action == ACTION.CANCEL:
+                            do_cancel()
+                            action = ACTION.NOOP
+                            return
+                        if has_gathering_completed():
+                            action = ACTION.CONCAT
+                            continue
+
+                    break
+
+            def on_completed() -> None:
+                if action == ACTION.GATHER:
+                    do_concat()
+                observer.on_completed()
+
+            def on_error(e: Exception) -> None:
+                if action == ACTION.GATHER:
+                    do_concat()
+                observer.on_error(e)
+
+            return source.subscribe(
+                on_next, on_error, on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    return _concat
+
+
+class JoinPointExtractor:
+    def __init__(self) -> None:
+        self._join_points: Subject[List[JoinPoint]] = Subject()
+
+    @property
+    def join_points(self) -> Observable[List[JoinPoint]]:
+        return self._join_points
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._extract(source)
+
+    def _extract(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            stream_index: int = -1
+            subscription = SerialDisposable()
+            join_points: List[JoinPoint] = []
+            join_point_tag: Optional[ScriptTag] = None
+
+            def push_join_points() -> None:
+                self._join_points.on_next(join_points.copy())
+
+            def on_next(item: FLVStreamItem) -> None:
+                nonlocal stream_index
+                nonlocal join_point_tag
+
+                if isinstance(item, FlvHeader):
+                    stream_index += 1
+                    if stream_index > 0:
+                        push_join_points()
+                        join_points.clear()
+                    join_point_tag = None
+                    observer.on_next(item)
+                    return
+
+                if join_point_tag:
+                    join_point = self._make_join_point(join_point_tag, item)
+                    join_points.append(join_point)
+                    logger.debug(f'{repr(join_point)}; {join_point}')
+                    join_point_tag = None
+
+                if self._is_join_point_tag(item):
+                    join_point_tag = item
+                    return
+
+                observer.on_next(item)
+
+            def on_completed() -> None:
+                push_join_points()
+                observer.on_completed()
+
+            def on_error(e: Exception) -> None:
+                push_join_points()
+                observer.on_error(e)
+
+            def dispose() -> None:
+                push_join_points()
+
+            subscription.disposable = source.subscribe(
+                on_next, on_error, on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)
+
+    def _is_join_point_tag(self, tag: FlvTag) -> TypeGuard[ScriptTag]:
+        if is_script_tag(tag):
+            script_data = parse_scriptdata(tag)
+            return script_data['name'] == 'onJoinPoint'
+        return False
+
+    def _make_join_point(self, join_point_tag: ScriptTag, tag: FlvTag) -> JoinPoint:
+        assert tag.body is not None
+        script_data = parse_scriptdata(join_point_tag)
+        join_point_data = cast(JoinPointData, script_data['value'])
+        return JoinPoint(
+            seamless=join_point_data['seamless'],
+            timestamp=tag.timestamp,
+            crc32=cksum(tag.body),
+        )

src/blrec/flv/operators/correct.py

@@ -0,0 +1,64 @@
+import logging
+from typing import Callable, Optional
+
+from reactivex import Observable, abc
+
+from ..common import is_script_tag, is_sequence_header
+from ..models import FlvHeader, FlvTag
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('correct',)
+
+logger = logging.getLogger(__name__)
+
+
+def correct() -> Callable[[FLVStream], FLVStream]:
+    def _correct(source: FLVStream) -> FLVStream:
+        """Correct the timestamp offset of the FLV tags."""
+
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            delta: Optional[int] = None
+
+            def correct_ts(tag: FlvTag, delta: int) -> FlvTag:
+                if delta == 0:
+                    return tag
+                return tag.evolve(timestamp=tag.timestamp + delta)
+
+            def on_next(item: FLVStreamItem) -> None:
+                nonlocal delta
+
+                if isinstance(item, FlvHeader):
+                    delta = None
+                    observer.on_next(item)
+                    return
+
+                tag = item
+
+                if is_script_tag(tag):
+                    tag = correct_ts(tag, -tag.timestamp)
+                    observer.on_next(tag)
+                    return
+
+                if delta is None:
+                    if is_sequence_header(tag):
+                        tag = correct_ts(tag, -tag.timestamp)
+                    else:
+                        logger.debug(f'The first data tag: {tag}')
+                        delta = -tag.timestamp
+                        logger.debug(f'Timestamp delta: {delta}')
+                        tag = correct_ts(tag, delta)
+                else:
+                    tag = correct_ts(tag, delta)
+
+                observer.on_next(tag)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    return _correct

src/blrec/flv/operators/cut.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from reactivex import Observable, abc
+
+from ..common import (
+    is_audio_sequence_header,
+    is_metadata_tag,
+    is_video_nalu_keyframe,
+    is_video_sequence_header,
+)
+from ..models import AudioTag, FlvHeader, FlvTag, ScriptTag, VideoTag
+from ..utils import format_timestamp
+from .correct import correct
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('Cutter',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Cutter:
+    def __init__(self, min_duration: int = 5_000) -> None:
+        self._min_duration = min_duration  # milliseconds
+        self._reset()
+
+    def _reset(self) -> None:
+        self._cutting: bool = False
+        self._triggered: bool = False
+        self._last_timestamp: int = 0
+        self._last_flv_header: Optional[FlvHeader] = None
+        self._last_metadata_tag: Optional[ScriptTag] = None
+        self._last_audio_sequence_header: Optional[AudioTag] = None
+        self._last_video_sequence_header: Optional[VideoTag] = None
+
+    def is_cutting(self) -> bool:
+        return self._cutting
+
+    def can_cut_stream(self) -> bool:
+        if self._triggered or self._cutting:
+            return False
+        return self._last_timestamp >= self._min_duration
+
+    def cut_stream(self) -> bool:
+        if self.can_cut_stream():
+            self._triggered = True
+            return True
+        return False
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._cut(source).pipe(correct())
+
+    def _cut(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_next(item: FLVStreamItem) -> None:
+                if isinstance(item, FlvHeader):
+                    self._reset()
+                    self._update_flv_header(item)
+                else:
+                    self._update_meta_tags(item)
+                    self._check_cut_point(item)
+                    if self._cutting:
+                        self._insert_header_and_tags(observer)
+                        self._cutting = False
+                        self._triggered = False
+                observer.on_next(item)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    def _update_flv_header(self, header: FlvHeader) -> None:
+        self._last_flv_header = header
+
+    def _update_meta_tags(self, tag: FlvTag) -> None:
+        if is_metadata_tag(tag):
+            self._last_metadata_tag = tag
+        elif is_audio_sequence_header(tag):
+            self._last_audio_sequence_header = tag
+        elif is_video_sequence_header(tag):
+            self._last_video_sequence_header = tag
+
+    def _insert_header_and_tags(
+        self, observer: abc.ObserverBase[FLVStreamItem]
+    ) -> None:
+        assert self._last_flv_header is not None
+        assert self._last_audio_sequence_header is not None
+        assert self._last_video_sequence_header is not None
+        observer.on_next(self._last_flv_header)
+        if self._last_metadata_tag is not None:
+            observer.on_next(self._last_metadata_tag)
+        observer.on_next(self._last_audio_sequence_header)
+        observer.on_next(self._last_video_sequence_header)
+
+    def _check_cut_point(self, tag: FlvTag) -> None:
+        self._last_timestamp = tag.timestamp
+
+        if not self._triggered:
+            return
+
+        if not is_video_nalu_keyframe(tag):
+            return
+
+        self._cutting = True
+        logger.info(f'Cut stream at {format_timestamp(tag.timestamp)}')

src/blrec/flv/operators/defragment.py

@@ -0,0 +1,56 @@
+import logging
+from typing import Callable, List, Optional
+
+from reactivex import Observable, abc
+
+from ..models import FlvHeader
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('defragment',)
+
+logger = logging.getLogger(__name__)
+
+
+def defragment(min_tags: int = 10) -> Callable[[FLVStream], FLVStream]:
+    def _defragment(source: FLVStream) -> FLVStream:
+        """Discard fragmented FLV streams."""
+
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            gathering: bool = False
+            gathered_items: List[FLVStreamItem] = []
+
+            def on_next(item: FLVStreamItem) -> None:
+                nonlocal gathering
+
+                if isinstance(item, FlvHeader):
+                    if gathered_items:
+                        logger.debug(
+                            'Discarded {} items, total size: {}'.format(
+                                len(gathered_items), sum(len(t) for t in gathered_items)
+                            )
+                        )
+                        gathered_items.clear()
+                    gathering = True
+                    logger.debug('Gathering items...')
+
+                if gathering:
+                    gathered_items.append(item)
+                    if len(gathered_items) > min_tags:
+                        for item in gathered_items:
+                            observer.on_next(item)
+                        gathered_items.clear()
+                        gathering = False
+                        logger.debug('Not a fragmented stream, stopped the gathering')
+                else:
+                    observer.on_next(item)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    return _defragment

src/blrec/flv/operators/dump.py

@@ -0,0 +1,123 @@
+import io
+import logging
+from typing import Callable, Optional, Tuple
+
+from reactivex import Observable, Subject, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+
+from ..io import FlvWriter
+from ..models import FlvHeader
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('Dumper',)
+
+logger = logging.getLogger(__name__)
+
+
+class Dumper:
+    def __init__(
+        self,
+        path_provider: Callable[..., Tuple[str, int]],
+        buffer_size: Optional[int] = None,
+    ) -> None:
+        self.buffer_size = buffer_size or io.DEFAULT_BUFFER_SIZE  # bytes
+        self._path_provider = path_provider
+        self._file_opened: Subject[Tuple[str, int]] = Subject()
+        self._file_closed: Subject[str] = Subject()
+        self._size_updates: Subject[int] = Subject()
+        self._timestamp_updates: Subject[int] = Subject()
+        self._reset()
+
+    def _reset(self) -> None:
+        self._path: str = ''
+        self._file: Optional[io.BufferedReader] = None
+        self._flv_writer: Optional[FlvWriter] = None
+
+    @property
+    def path(self) -> str:
+        return self._path
+
+    @property
+    def file_opened(self) -> Observable[Tuple[str, int]]:
+        return self._file_opened
+
+    @property
+    def file_closed(self) -> Observable[str]:
+        return self._file_closed
+
+    @property
+    def size_updates(self) -> Observable[int]:
+        return self._size_updates
+
+    @property
+    def timestamp_updates(self) -> Observable[int]:
+        return self._timestamp_updates
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._dump(source)
+
+    def _open_file(self) -> None:
+        self._path, timestamp = self._path_provider()
+        self._file = open(self._path, 'wb', buffering=self.buffer_size)  # type: ignore
+        logger.debug(f'Opened file: {self._path}')
+        self._file_opened.on_next((self._path, timestamp))
+
+    def _close_file(self) -> None:
+        if self._file is not None and not self._file.closed:
+            self._file.close()
+            logger.debug(f'Closed file: {self._path}')
+            self._file_closed.on_next(self._path)
+
+    def _dump(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            subscription = SerialDisposable()
+
+            self._close_file()
+            self._reset()
+
+            def on_next(item: FLVStreamItem) -> None:
+                try:
+                    if isinstance(item, FlvHeader):
+                        self._close_file()
+                        self._open_file()
+                        assert self._file is not None
+                        self._flv_writer = FlvWriter(self._file)
+                        size = self._flv_writer.write_header(item)
+                        self._size_updates.on_next(size)
+                        self._timestamp_updates.on_next(0)
+                    else:
+                        if self._flv_writer is not None:
+                            size = self._flv_writer.write_tag(item)
+                            self._size_updates.on_next(size)
+                            self._timestamp_updates.on_next(item.timestamp)
+
+                    observer.on_next(item)
+                except Exception as e:
+                    self._close_file()
+                    self._reset()
+                    observer.on_error(e)
+
+            def on_completed() -> None:
+                self._close_file()
+                self._reset()
+                observer.on_completed()
+
+            def on_error(e: Exception) -> None:
+                self._close_file()
+                self._reset()
+                observer.on_error(e)
+
+            def dispose() -> None:
+                self._close_file()
+                self._reset()
+
+            subscription.disposable = source.subscribe(
+                on_next, on_error, on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)

src/blrec/flv/operators/fix.py

@@ -0,0 +1,152 @@
+import logging
+import math
+from typing import Callable, Optional
+
+from reactivex import Observable, abc
+
+from ..common import (
+    is_audio_tag,
+    is_metadata_tag,
+    is_script_tag,
+    is_video_tag,
+    parse_metadata,
+)
+from ..models import AudioTag, FlvHeader, FlvTag, ScriptTag, VideoTag
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('fix',)
+
+logger = logging.getLogger(__name__)
+
+
+def fix() -> Callable[[FLVStream], FLVStream]:
+    def _fix(source: FLVStream) -> FLVStream:
+        """Fix broken timestamps of the FLV tags."""
+
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            delta: int = 0
+            last_tag: Optional[FlvTag] = None
+            last_audio_tag: Optional[AudioTag] = None
+            last_video_tag: Optional[VideoTag] = None
+            frame_rate = 30.0
+            video_frame_interval = math.ceil(1000 / frame_rate)
+            sound_sample_interval = math.ceil(1000 / 44)
+
+            def reset() -> None:
+                nonlocal delta, last_tag, last_audio_tag, last_video_tag
+                nonlocal frame_rate, video_frame_interval, sound_sample_interval
+                delta = 0
+                last_tag = None
+                last_audio_tag = None
+                last_video_tag = None
+                frame_rate = 30.0
+                video_frame_interval = math.ceil(1000 / frame_rate)
+                sound_sample_interval = math.ceil(1000 / 44)
+
+            def update_parameters(tag: ScriptTag) -> None:
+                nonlocal frame_rate, video_frame_interval
+                metadata = parse_metadata(tag)
+                fps = metadata.get('fps') or metadata.get('framerate')
+
+                if not fps:
+                    return
+
+                frame_rate = fps
+                video_frame_interval = math.ceil(1000 / frame_rate)
+
+                logger.debug(
+                    'frame rate: {}, video frame interval: {}'.format(
+                        frame_rate, video_frame_interval
+                    )
+                )
+
+            def update_last_tags(tag: FlvTag) -> None:
+                nonlocal last_tag, last_audio_tag, last_video_tag
+                last_tag = tag
+                if is_audio_tag(tag):
+                    last_audio_tag = tag
+                elif is_video_tag(tag):
+                    last_video_tag = tag
+
+            def update_delta(tag: FlvTag) -> None:
+                nonlocal delta
+                assert last_tag is not None
+                delta = last_tag.timestamp + delta - tag.timestamp + calc_interval(tag)
+
+            def correct_ts(tag: FlvTag) -> FlvTag:
+                if delta == 0:
+                    return tag
+                return tag.evolve(timestamp=tag.timestamp + delta)
+
+            def calc_interval(tag: FlvTag) -> int:
+                if is_audio_tag(tag):
+                    return sound_sample_interval
+                elif is_video_tag(tag):
+                    return video_frame_interval
+                else:
+                    logger.warning(f'Unexpected tag type: {tag}')
+                    return min(sound_sample_interval, video_frame_interval)
+
+            def is_ts_rebounded(tag: FlvTag) -> bool:
+                if is_audio_tag(tag):
+                    if last_audio_tag is None:
+                        return False
+                    return tag.timestamp < last_audio_tag.timestamp
+                elif is_video_tag(tag):
+                    if last_video_tag is None:
+                        return False
+                    return tag.timestamp < last_video_tag.timestamp
+                else:
+                    return False
+
+            def is_ts_incontinuous(tag: FlvTag) -> bool:
+                if last_tag is None:
+                    return False
+                return tag.timestamp - last_tag.timestamp > max(
+                    sound_sample_interval, video_frame_interval
+                )
+
+            def on_next(item: FLVStreamItem) -> None:
+                if isinstance(item, FlvHeader):
+                    reset()
+                    observer.on_next(item)
+                    return
+
+                tag = item
+
+                if is_script_tag(tag):
+                    if is_metadata_tag(tag):
+                        update_parameters(tag)
+                    observer.on_next(tag)
+                    return
+
+                if is_ts_rebounded(tag):
+                    update_delta(tag)
+                    logger.warning(
+                        f'Timestamp rebounded, updated delta: {delta}\n'
+                        f'last audio tag: {last_audio_tag}\n'
+                        f'last video tag: {last_video_tag}\n'
+                        f'current tag: {tag}'
+                    )
+                elif is_ts_incontinuous(tag):
+                    update_delta(tag)
+                    logger.warning(
+                        f'Timestamp incontinuous, updated delta: {delta}\n'
+                        f'last tag: {last_tag}\n'
+                        f'current tag: {tag}'
+                    )
+
+                update_last_tags(tag)
+                tag = correct_ts(tag)
+                observer.on_next(tag)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    return _fix

src/blrec/flv/operators/helpers.py

@@ -0,0 +1,38 @@
+import io
+import logging
+
+from reactivex import of
+
+from .parse import parse
+from .typing import FLVStream
+
+__all__ = ('from_file', 'from_stream')
+
+logger = logging.getLogger(__name__)
+
+
+def from_stream(
+    stream: io.RawIOBase,
+    *,
+    complete_on_eof: bool = False,
+    backup_timestamp: bool = False,
+    restore_timestamp: bool = False,
+) -> FLVStream:
+    return of(stream).pipe(
+        parse(
+            complete_on_eof=complete_on_eof,
+            backup_timestamp=backup_timestamp,
+            restore_timestamp=restore_timestamp,
+        )
+    )
+
+
+def from_file(
+    path: str, *, backup_timestamp: bool = False, restore_timestamp: bool = False
+) -> FLVStream:
+    return from_stream(
+        open(path, 'rb'),
+        complete_on_eof=True,
+        backup_timestamp=backup_timestamp,
+        restore_timestamp=restore_timestamp,
+    )

src/blrec/flv/operators/inject.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+import logging
+from typing import Any, Callable, Dict, Optional, cast
+
+from reactivex import Observable, abc
+
+from ..common import (
+    create_metadata_tag,
+    enrich_metadata,
+    is_metadata_tag,
+    parse_metadata,
+    update_metadata,
+)
+from ..models import FlvHeader, ScriptTag
+from .analyse import KeyFramesDict
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('Injector',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Injector:
+    def __init__(self, metadata_provider: Callable[..., Dict[str, Any]]) -> None:
+        self._metadata_provider = metadata_provider
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._inject(source)
+
+    def _inject(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            index = 0
+
+            def on_next(item: FLVStreamItem) -> None:
+                nonlocal index
+                if isinstance(item, FlvHeader):
+                    index = 0
+                    observer.on_next(item)
+                    return
+
+                tag = item
+                index += 1
+
+                if index == 1:
+                    if is_metadata_tag(tag):
+                        tag = self._inject_metadata(tag)
+                        logger.debug('Injected metadata into the metadata tag')
+                    else:
+                        logger.debug('No metadata tag in the stream')
+                        tag = self._make_metadata_tag()
+                        logger.debug('Maked a metadata tag for metadata injection')
+                        observer.on_next(tag)
+                        logger.debug('Inserted the artificial metadata tag')
+
+                observer.on_next(tag)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    def _inject_metadata(self, tag: ScriptTag) -> ScriptTag:
+        old_metadata = parse_metadata(tag)
+        new_metadata = self._metadata_provider()
+        final_metadata = {
+            **{'duration': 0.0, 'filesize': 0.0},
+            **old_metadata,
+            **new_metadata,
+        }
+        new_tag = enrich_metadata(tag, final_metadata, offset=tag.offset)
+
+        if 'keyframes' in final_metadata:
+            keyframes = cast(KeyFramesDict, final_metadata['keyframes'])
+            offset = new_tag.tag_size - tag.tag_size
+            keyframes['filepositions'] = list(
+                map(lambda p: p + offset, keyframes['filepositions'])
+            )
+            if 'lastkeyframelocation' in final_metadata:
+                final_metadata['lastkeyframelocation'] = keyframes['filepositions'][-1]
+            new_tag = update_metadata(new_tag, final_metadata)
+
+        return new_tag
+
+    def _make_metadata_tag(self) -> ScriptTag:
+        metadata = self._metadata_provider()
+        metadata = {'duration': 0.0, 'filesize': 0.0, **metadata}
+        return create_metadata_tag(metadata)

src/blrec/flv/operators/limit.py

@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from reactivex import Observable, abc
+
+from ..common import (
+    is_audio_sequence_header,
+    is_metadata_tag,
+    is_video_nalu_keyframe,
+    is_video_sequence_header,
+)
+from ..models import BACK_POINTER_SIZE, AudioTag, FlvHeader, FlvTag, ScriptTag, VideoTag
+from .correct import correct
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('Limiter',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class Limiter:
+    def __init__(
+        self,
+        filesize_limit: int = 0,  # file size in bytes, no limit by default.
+        duration_limit: int = 0,  # duration in seconds, no limit by default.
+    ) -> None:
+        self.filesize_limit = filesize_limit
+        self.duration_limit = duration_limit
+        self._reset()
+
+    def _reset(self) -> None:
+        self._filesize: int = 0
+        self._duration: float = 0.0
+        self._max_size_between_keyframes: int = 0
+        self._max_duration_between_keyframes: float = 0.0
+        self._first_keyframe_tag: Optional[VideoTag] = None
+        self._last_keyframe_tag: Optional[VideoTag] = None
+        self._last_flv_header: Optional[FlvHeader] = None
+        self._last_metadata_tag: Optional[ScriptTag] = None
+        self._last_audio_sequence_header: Optional[AudioTag] = None
+        self._last_video_sequence_header: Optional[VideoTag] = None
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._limit(source).pipe(correct())
+
+    def _limit(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            def on_next(item: FLVStreamItem) -> None:
+                if isinstance(item, FlvHeader):
+                    self._reset()
+                    self._update_flv_header(item)
+                else:
+                    self._update_meta_tags(item)
+                    will_over_limts = self._check_limits(item)
+                    if will_over_limts:
+                        self._insert_header_and_tags(observer)
+                observer.on_next(item)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    def _insert_header_and_tags(
+        self, observer: abc.ObserverBase[FLVStreamItem]
+    ) -> None:
+        assert self._last_flv_header is not None
+        assert self._last_audio_sequence_header is not None
+        assert self._last_video_sequence_header is not None
+        observer.on_next(self._last_flv_header)
+        if self._last_metadata_tag is not None:
+            observer.on_next(self._last_metadata_tag)
+        observer.on_next(self._last_audio_sequence_header)
+        observer.on_next(self._last_video_sequence_header)
+
+        self._filesize = (
+            self._last_flv_header.size
+            + self._last_audio_sequence_header.tag_size
+            + self._last_video_sequence_header.tag_size
+        )
+        if self._last_metadata_tag is not None:
+            self._filesize += self._last_metadata_tag.tag_size
+        self._duration = 0.0
+        self._first_keyframe_tag = self._last_keyframe_tag
+
+    def _will_filesize_over_limit(self) -> bool:
+        return self._filesize + self._max_size_between_keyframes >= self.filesize_limit
+
+    def _will_duration_over_limit(self) -> bool:
+        return (
+            self._duration + self._max_duration_between_keyframes >= self.duration_limit
+        )
+
+    def _update_flv_header(self, header: FlvHeader) -> None:
+        self._filesize += header.size + BACK_POINTER_SIZE
+        self._last_flv_header = header
+
+    def _update_meta_tags(self, tag: FlvTag) -> None:
+        if is_metadata_tag(tag):
+            self._last_metadata_tag = tag
+        elif is_audio_sequence_header(tag):
+            self._last_audio_sequence_header = tag
+        elif is_video_sequence_header(tag):
+            self._last_video_sequence_header = tag
+
+    def _check_limits(self, tag: FlvTag) -> bool:
+        self._filesize += tag.tag_size + BACK_POINTER_SIZE
+
+        if not is_video_nalu_keyframe(tag):
+            return False
+
+        if self._first_keyframe_tag is None:
+            self._first_keyframe_tag = tag
+
+        if self._last_keyframe_tag is not None:
+            self._max_size_between_keyframes = max(
+                self._max_size_between_keyframes,
+                tag.offset - self._last_keyframe_tag.offset,
+            )
+            self._max_duration_between_keyframes = max(
+                self._max_duration_between_keyframes,
+                (tag.timestamp - self._last_keyframe_tag.timestamp) / 1000,
+            )
+
+        self._last_keyframe_tag = tag
+        self._duration = (
+            self._last_keyframe_tag.timestamp - self._first_keyframe_tag.timestamp
+        ) / 1000
+
+        if self.filesize_limit > 0 and self._will_filesize_over_limit():
+            logger.debug(
+                'File size will be over the limit: {} + {}'.format(
+                    self._filesize, self._max_size_between_keyframes
+                )
+            )
+            return True
+
+        if self.duration_limit > 0 and self._will_duration_over_limit():
+            logger.debug(
+                'Duration will be over the limit: {} + {}'.format(
+                    self._duration, self._max_duration_between_keyframes
+                )
+            )
+            return True
+
+        return False

src/blrec/flv/operators/parse.py

@@ -0,0 +1,68 @@
+import io
+import logging
+from typing import Callable, Optional
+
+from reactivex import Observable, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+
+from ..io import FlvReader
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('parse',)
+
+logger = logging.getLogger(__name__)
+
+
+def parse(
+    *,
+    ignore_eof: bool = False,
+    complete_on_eof: bool = False,
+    backup_timestamp: bool = False,
+    restore_timestamp: bool = False,
+) -> Callable[[Observable[io.RawIOBase]], FLVStream]:
+    def _parse(source: Observable[io.RawIOBase]) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            disposed = False
+            subscription = SerialDisposable()
+
+            def on_next(stream: io.RawIOBase) -> None:
+                try:
+                    try:
+                        reader = FlvReader(
+                            stream,
+                            backup_timestamp=backup_timestamp,
+                            restore_timestamp=restore_timestamp,
+                        )
+                        observer.on_next(reader.read_header())
+                        while not disposed:
+                            tag = reader.read_tag()
+                            observer.on_next(tag)
+                    finally:
+                        stream.close()
+                except EOFError as e:
+                    if complete_on_eof:
+                        observer.on_completed()
+                    else:
+                        if not ignore_eof:
+                            observer.on_error(e)
+                except Exception as e:
+                    observer.on_error(e)
+                else:
+                    observer.on_completed()
+
+            subscription.disposable = source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+            def dispose() -> None:
+                nonlocal disposed
+                disposed = True
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)
+
+    return _parse

src/blrec/flv/operators/probe.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import io
+import logging
+from typing import List, Optional, cast
+
+from reactivex import Observable, Subject, abc
+
+from ...utils.ffprobe import StreamProfile, ffprobe
+from ..io import FlvWriter
+from ..models import FlvHeader, FlvTag
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('Prober', 'StreamProfile')
+
+
+logger = logging.getLogger(__name__)
+
+
+class Prober:
+    def __init__(self) -> None:
+        self._profiles: Subject[StreamProfile] = Subject()
+
+    def _reset(self) -> None:
+        self._gathering: bool = False
+        self._gathered_items: List[FLVStreamItem] = []
+
+    @property
+    def profiles(self) -> Observable[StreamProfile]:
+        return self._profiles
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._probe(source)
+
+    def _probe(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            self._reset()
+
+            def on_next(item: FLVStreamItem) -> None:
+                if isinstance(item, FlvHeader):
+                    self._gathered_items.clear()
+                    self._gathering = True
+
+                if self._gathering:
+                    self._gathered_items.append(item)
+                    if len(self._gathered_items) >= 10:
+                        try:
+                            self._do_probe()
+                        except Exception as e:
+                            logger.warning(f'Failed to probe stream: {repr(e)}')
+                        finally:
+                            self._gathered_items.clear()
+                            self._gathering = False
+
+                observer.on_next(item)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe)
+
+    def _do_probe(self) -> None:
+        bytes_io = io.BytesIO()
+        writer = FlvWriter(bytes_io)
+        writer.write_header(cast(FlvHeader, self._gathered_items[0]))
+        for tag in self._gathered_items[1:]:
+            writer.write_tag(cast(FlvTag, tag))
+
+        def on_next(profile: StreamProfile) -> None:
+            self._profiles.on_next(profile)
+
+        def on_error(e: Exception) -> None:
+            logger.warning(f'Failed to probe stream by ffprobe: {repr(e)}')
+
+        ffprobe(bytes_io.getvalue()).subscribe(on_next, on_error)

src/blrec/flv/operators/process.py

@@ -0,0 +1,19 @@
+import logging
+from typing import Callable
+
+from .concat import concat
+from .defragment import defragment
+from .fix import fix
+from .split import split
+from .typing import FLVStream
+
+__all__ = ('process',)
+
+logger = logging.getLogger(__name__)
+
+
+def process() -> Callable[[FLVStream], FLVStream]:
+    def _process(source: FLVStream) -> FLVStream:
+        return source.pipe(defragment(), split(), fix(), concat())
+
+    return _process

src/blrec/flv/operators/progress.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from reactivex import Observable, abc
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+from tqdm import tqdm
+
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('ProgressBar',)
+
+
+logger = logging.getLogger(__name__)
+
+
+class ProgressBar:
+    def __init__(
+        self,
+        desc: str,
+        postfix: Optional[str] = None,
+        total: Optional[int] = None,
+        disable: Optional[bool] = False,
+    ) -> None:
+        self._desc = desc
+        self._postfix = postfix
+        self._total = total
+        self._disable = disable
+        self._pbar: Optional[tqdm] = None
+
+    def __call__(self, source: FLVStream) -> FLVStream:
+        return self._progress(source)
+
+    def _progress(self, source: FLVStream) -> FLVStream:
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            subscription = SerialDisposable()
+
+            self._pbar = tqdm(
+                disable=self._disable,
+                desc=self._desc,
+                total=self._total,
+                unit='B',
+                unit_scale=True,
+                unit_divisor=1024,
+                postfix=self._postfix,
+            )
+            self._pbar.disable
+
+            def on_next(item: FLVStreamItem) -> None:
+                if self._pbar is not None:
+                    self._pbar.update(len(item))
+                observer.on_next(item)
+
+            def on_completed() -> None:
+                if self._pbar is not None:
+                    self._pbar.close()
+                    self._pbar = None
+                observer.on_completed()
+
+            def dispose() -> None:
+                if self._pbar is not None:
+                    self._pbar.close()
+                    self._pbar = None
+
+            subscription.disposable = source.subscribe(
+                on_next, observer.on_error, on_completed, scheduler=scheduler
+            )
+
+            return CompositeDisposable(subscription, Disposable(dispose))
+
+        return Observable(subscribe)

src/blrec/flv/operators/split.py

@@ -0,0 +1,100 @@
+import logging
+from typing import Callable, Optional
+
+from reactivex import Observable, abc
+
+from ..common import is_audio_sequence_header, is_metadata_tag, is_video_sequence_header
+from ..models import AudioTag, FlvHeader, ScriptTag, VideoTag
+from .correct import correct
+from .typing import FLVStream, FLVStreamItem
+
+__all__ = ('split',)
+
+logger = logging.getLogger(__name__)
+
+
+def split() -> Callable[[FLVStream], FLVStream]:
+    def _split(source: FLVStream) -> FLVStream:
+        """Split the FLV stream when av parameters are changed."""
+
+        def subscribe(
+            observer: abc.ObserverBase[FLVStreamItem],
+            scheduler: Optional[abc.SchedulerBase] = None,
+        ) -> abc.DisposableBase:
+            changed: bool = False
+            last_flv_header: Optional[FlvHeader] = None
+            last_metadata_tag: Optional[ScriptTag] = None
+            last_audio_sequence_header: Optional[AudioTag] = None
+            last_video_sequence_header: Optional[VideoTag] = None
+
+            def reset() -> None:
+                nonlocal changed
+                nonlocal last_flv_header
+                nonlocal last_metadata_tag
+                nonlocal last_audio_sequence_header, last_video_sequence_header
+                changed = False
+                last_flv_header = None
+                last_metadata_tag = None
+                last_audio_sequence_header = last_video_sequence_header = None
+
+            def insert_header_and_tags() -> None:
+                assert last_flv_header is not None
+                assert last_audio_sequence_header is not None
+                assert last_video_sequence_header is not None
+                observer.on_next(last_flv_header)
+                if last_metadata_tag is not None:
+                    observer.on_next(last_metadata_tag)
+                observer.on_next(last_audio_sequence_header)
+                observer.on_next(last_video_sequence_header)
+
+            def on_next(item: FLVStreamItem) -> None:
+                nonlocal changed
+                nonlocal last_flv_header
+                nonlocal last_metadata_tag
+                nonlocal last_audio_sequence_header, last_video_sequence_header
+
+                if isinstance(item, FlvHeader):
+                    reset()
+                    last_flv_header = item
+                    observer.on_next(item)
+                    return
+
+                tag = item
+
+                if is_metadata_tag(tag):
+                    logger.debug(f'Metadata tag: {tag}')
+                    last_metadata_tag = tag
+                elif is_audio_sequence_header(tag):
+                    logger.debug(f'Audio sequence header: {tag}')
+                    if last_audio_sequence_header is not None:
+                        if not tag.is_the_same_as(last_audio_sequence_header):
+                            logger.warning('Audio parameters changed')
+                            changed = True
+                        last_audio_sequence_header = tag
+                        return
+                    last_audio_sequence_header = tag
+                elif is_video_sequence_header(tag):
+                    logger.debug(f'Video sequence header: {tag}')
+                    if last_video_sequence_header is not None:
+                        if not tag.is_the_same_as(last_video_sequence_header):
+                            logger.warning('Video parameters changed')
+                            changed = True
+                        last_video_sequence_header = tag
+                        return
+                    last_video_sequence_header = tag
+                else:
+                    if changed:
+                        logger.debug('Splitting stream...')
+                        changed = False
+                        insert_header_and_tags()
+                        logger.debug('Splitted stream')
+
+                observer.on_next(tag)
+
+            return source.subscribe(
+                on_next, observer.on_error, observer.on_completed, scheduler=scheduler
+            )
+
+        return Observable(subscribe).pipe(correct())
+
+    return _split

src/blrec/flv/operators/typing.py

@@ -0,0 +1,8 @@
+from typing import Union
+
+from reactivex import Observable
+
+from ..models import FlvHeader, FlvTag
+
+FLVStreamItem = Union[FlvHeader, FlvTag]
+FLVStream = Observable[FLVStreamItem]

src/blrec/flv/parameters_checker.py

@@ -1,58 +0,0 @@
-from __future__ import annotations
-import logging
-from typing import Optional
-
-
-from .models import AudioTag, FlvTag, ScriptTag, VideoTag
-from .common import (
-    is_audio_sequence_header, is_video_sequence_header, is_metadata_tag
-)
-from .exceptions import AudioParametersChanged, VideoParametersChanged
-
-
-__all__ = 'ParametersChecker',
-
-
-logger = logging.getLogger(__name__)
-
-
-class ParametersChecker:
-    def __init__(self) -> None:
-        self.reset()
-
-    @property
-    def last_metadata_tag(self) -> Optional[ScriptTag]:
-        return self._last_metadata_tag
-
-    @property
-    def last_audio_header_tag(self) -> Optional[AudioTag]:
-        return self._last_audio_header_tag
-
-    @property
-    def last_video_header_tag(self) -> Optional[VideoTag]:
-        return self._last_video_header_tag
-
-    def reset(self) -> None:
-        self._last_metadata_tag: Optional[ScriptTag] = None
-        self._last_audio_header_tag: Optional[AudioTag] = None
-        self._last_video_header_tag: Optional[VideoTag] = None
-
-    def check_tag(self, tag: FlvTag) -> None:
-        if is_audio_sequence_header(tag):
-            if self._last_audio_header_tag is not None:
-                if not tag.is_the_same_as(self._last_audio_header_tag):
-                    logger.debug(f'Audio parameters changed: {tag}')
-                    self._last_audio_header_tag = tag
-                    raise AudioParametersChanged()
-            self._last_audio_header_tag = tag
-        elif is_video_sequence_header(tag):
-            if self._last_video_header_tag is not None:
-                if not tag.is_the_same_as(self._last_video_header_tag):
-                    logger.debug(f'Video parameters changed: {tag}')
-                    self._last_video_header_tag = tag
-                    raise VideoParametersChanged()
-            self._last_video_header_tag = tag
-        elif is_metadata_tag(tag):
-            self._last_metadata_tag = tag
-        else:
-            pass

src/blrec/flv/stream_cutter.py

@@ -1,60 +0,0 @@
-from __future__ import annotations
-import logging
-from typing import Optional
-
-
-from .models import FlvTag, VideoTag
-from .exceptions import CutStream
-from .common import is_video_nalu_keyframe
-from .utils import format_timestamp
-
-
-__all__ = 'StreamCutter',
-
-
-logger = logging.getLogger(__name__)
-
-
-class StreamCutter:
-    def __init__(self, min_duration: int = 5_000) -> None:
-        self._min_duration = min_duration  # milliseconds
-        self._last_position: int = 0
-        self.reset()
-
-    @property
-    def last_keyframe_tag(self) -> Optional[VideoTag]:
-        return self._last_keyframe_tag
-
-    def is_cutting(self) -> bool:
-        return self._cutting
-
-    def can_cut_stream(self) -> bool:
-        return self._timestamp >= self._min_duration
-
-    def cut_stream(self) -> bool:
-        if self.can_cut_stream():
-            self._triggered = True
-            return True
-        return False
-
-    def reset(self) -> None:
-        self._cutting = False
-        self._triggered = False
-        self._timestamp: int = 0
-        self._last_keyframe_tag: Optional[VideoTag] = None
-
-    def check_tag(self, tag: FlvTag) -> None:
-        self._timestamp = tag.timestamp
-
-        if not self._triggered:
-            return
-
-        if not is_video_nalu_keyframe(tag):
-            return
-
-        self._last_keyframe_tag = tag
-        self._last_position += self._timestamp
-        self._cutting = True
-
-        logger.info(f'Cut stream at: {format_timestamp(self._last_position)}')
-        raise CutStream()

src/blrec/flv/stream_processor.py

@@ -1,891 +0,0 @@
-from __future__ import annotations
-import io
-import math
-from abc import ABC, abstractmethod
-import json
-import logging
-from typing import (
-    Any, BinaryIO, Dict, List, Final, Iterable, Iterator, Optional, Tuple,
-    Protocol, TypedDict, Union, cast, TYPE_CHECKING
-)
-
-import attr
-from rx.subject import Subject
-from rx.core import Observable
-
-from .models import FlvHeader, FlvTag, ScriptTag, VideoTag, AudioTag
-from .data_analyser import DataAnalyser, MetaData
-from .stream_cutter import StreamCutter
-from .limit_checker import LimitChecker
-from .parameters_checker import ParametersChecker
-from .io import FlvReader, FlvWriter
-from .io_protocols import RandomIO
-from .utils import format_offest, format_timestamp
-from .exceptions import (
-    FlvTagError,
-    FlvStreamCorruptedError,
-    AudioParametersChanged,
-    VideoParametersChanged,
-    FileSizeOverLimit,
-    DurationOverLimit,
-    CutStream,
-)
-from .common import (
-    is_audio_tag, is_video_tag, is_metadata_tag, parse_metadata,
-    is_audio_data_tag, is_video_data_tag, enrich_metadata, update_metadata,
-    is_data_tag, read_tags_in_duration,
-)
-from ..path import extra_metadata_path
-if TYPE_CHECKING:
-    from ..core.stream_analyzer import StreamProfile
-
-
-__all__ = 'StreamProcessor', 'BaseOutputFileManager', 'JoinPoint'
-
-
-logger = logging.getLogger(__name__)
-
-
-class StreamProcessor:
-    # Number of tags for determining whether or not tags are duplicated
-    _TAG_SEQUENCE_COUNT: Final[int] = 3
-    # Max duration in milliseconds the duplicated tags might last
-    _MAX_DURATION: Final[int] = 20000
-
-    def __init__(
-        self,
-        file_manager: OutputFileManager,
-        *,
-        metadata: Optional[Dict[str, Any]] = None,
-        filesize_limit: int = 0,
-        duration_limit: int = 0,
-        disable_limit: bool = False,
-        analyse_data: bool = False,
-        dedup_join: bool = False,
-        save_extra_metadata: bool = False,
-    ) -> None:
-        self._file_manager = file_manager
-        self._parameters_checker = ParametersChecker()
-        self._stream_cutter = StreamCutter()
-        if not disable_limit:
-            self._limit_checker = LimitChecker(filesize_limit, duration_limit)
-        if analyse_data:
-            self._data_analyser = DataAnalyser()
-
-        self._metadata = metadata.copy() if metadata else {}
-        self._metadata_tag: ScriptTag
-
-        self._disable_limit = disable_limit
-        self._analyse_data = analyse_data
-        self._dedup_join = dedup_join
-        self._save_x_metadata = save_extra_metadata
-
-        self._cancelled: bool = False
-        self._finalized: bool = False
-        self._stream_count: int = 0
-        self._size_updates = Subject()
-        self._time_updates = Subject()
-        self._stream_profile_updates = Subject()
-
-        self._delta: int = 0
-        self._has_audio: bool = False
-        self._last_tags: List[FlvTag] = []
-        self._join_points: List[JoinPoint] = []
-        self._resetting_file: bool = False
-
-    @property
-    def filesize_limit(self) -> int:
-        if self._disable_limit:
-            return 0
-        return self._limit_checker.filesize_limit
-
-    @filesize_limit.setter
-    def filesize_limit(self, value: int) -> None:
-        if not self._disable_limit:
-            self._limit_checker.filesize_limit = value
-
-    @property
-    def duration_limit(self) -> int:
-        if self._disable_limit:
-            return 0
-        return self._limit_checker.duration_limit
-
-    @duration_limit.setter
-    def duration_limit(self, value: int) -> None:
-        if not self._disable_limit:
-            self._limit_checker.duration_limit = value
-
-    @property
-    def join_points(self) -> Iterator[JoinPoint]:
-        for point in self._join_points:
-            yield point
-
-    @property
-    def metadata(self) -> Optional[MetaData]:
-        if not self._analyse_data:
-            return None
-        try:
-            return self._data_analyser.make_metadata()
-        except AssertionError:
-            return None
-        except Exception as e:
-            logger.debug(f'Failed to make metadata data, due to: {repr(e)}')
-            return None
-
-    @property
-    def size_updates(self) -> Observable:
-        return self._size_updates
-
-    @property
-    def time_updates(self) -> Observable:
-        return self._time_updates
-
-    @property
-    def stream_profile_updates(self) -> Observable:
-        return self._stream_profile_updates
-
-    @property
-    def cancelled(self) -> bool:
-        return self._cancelled
-
-    @property
-    def finalized(self) -> bool:
-        return self._finalized
-
-    def cancel(self) -> None:
-        self._cancelled = True
-
-    def set_metadata(self, metadata: Dict[str, Any]) -> None:
-        self._metadata = metadata.copy()
-
-    def process_stream(self, stream: RandomIO) -> None:
-        assert not self._cancelled and not self._finalized, \
-            'should not be called after the processing cancelled or finalized'
-        self._stream_count += 1
-        self._process_stream(stream)
-
-    def can_cut_stream(self) -> bool:
-        return self._stream_cutter.can_cut_stream()
-
-    def cut_stream(self) -> bool:
-        return self._stream_cutter.cut_stream()
-
-    def finalize(self) -> None:
-        assert not self._finalized, \
-            'should not be called after the processing finalized'
-        self._finalized = True
-
-        if not self._need_to_finalize():
-            logger.debug('No need to finalize stream processing')
-            return
-
-        self._complete_file()
-        logger.debug('Finalized stream processing')
-
-    def _need_to_finalize(self) -> bool:
-        return self._stream_count > 0 and len(self._last_tags) > 0
-
-    def _reset_params(self) -> None:
-        self._delta = 0
-        self._has_audio = False
-        self._last_tags = []
-        self._join_points = []
-        self._resetting_file = False
-
-        self._stream_cutter.reset()
-        if not self._disable_limit:
-            self._limit_checker.reset()
-        if self._analyse_data:
-            self._data_analyser.reset()
-
-    def _new_file(self) -> None:
-        self._reset_params()
-        self._out_file = self._file_manager.create_file()
-        self._out_writer = FlvWriter(self._out_file)
-        logger.debug(f'New file: {self._file_manager.curr_path}')
-
-    def _reset_file(self) -> None:
-        self._reset_params()
-        self._out_file.truncate(0)
-        logger.debug(f'Reset file: {self._file_manager.curr_path}')
-
-    def _complete_file(self) -> None:
-        curr_path = self._file_manager.curr_path
-
-        if self._save_x_metadata:
-            self._save_extra_metadata()
-
-        self._update_metadata_tag()
-        self._file_manager.close_file()
-
-        logger.debug(f'Complete file: {curr_path}')
-
-    def _process_stream(self, stream: RandomIO) -> None:
-        logger.debug(f'Processing the {self._stream_count}th stream...')
-
-        self._in_reader = FlvReaderWithTimestampFix(stream)
-        flv_header = self._read_header()
-        self._has_audio = flv_header.has_audio()
-
-        try:
-            first_data_tag = self._read_first_data_tag()
-            if not self._last_tags:
-                self._process_initial_stream(flv_header, first_data_tag)
-            else:
-                self._process_subsequent_stream(first_data_tag)
-        except (
-            AudioParametersChanged, VideoParametersChanged,
-            FileSizeOverLimit, DurationOverLimit, CutStream,
-        ):
-            self._process_split_stream(flv_header)
-
-        logger.debug(f'Completed processing the {self._stream_count}th stream')
-
-    def _process_initial_stream(
-        self, flv_header: FlvHeader, first_data_tag: FlvTag
-    ) -> None:
-        if self._resetting_file:
-            self._reset_file()
-        else:
-            self._new_file()
-
-        try:
-            self._write_header(self._ensure_header_correct(flv_header))
-            self._transfer_meta_tags()
-            self._transfer_first_data_tag(first_data_tag)
-            self._update_stream_profile(flv_header, first_data_tag)
-        except Exception:
-            self._last_tags = []
-            self._resetting_file = True
-            raise
-        else:
-            del flv_header, first_data_tag
-
-        self._transfer_tags_until_complete()
-
-    def _process_subsequent_stream(self, first_data_tag: FlvTag) -> None:
-        tags: List[FlvTag] = []
-
-        if self._dedup_join:
-            tags, exc = self._read_tags_for_deduplication()
-
-            if (index := self._find_last_duplicated_tag(tags)) >= 0:
-                seamless = True
-                self._delta = self._calc_delta_duplicated(tags[index])
-                tags = tags[index + 1:]
-                if not tags:
-                    tags = [self._read_first_data_tag()]
-
-        if not self._dedup_join or index == -1:
-            seamless = False
-            self._delta = self._calc_delta_no_duplicated(first_data_tag)
-            tags.insert(0, first_data_tag)
-
-        offset = self._out_file.tell()
-        timestamp = tags[0].timestamp + self._delta
-        self._add_join_point(offset, timestamp, seamless)
-
-        self._transfer_tags(tags)
-        del first_data_tag, tags
-
-        if self._dedup_join and exc:
-            raise exc
-
-        self._transfer_tags_until_complete()
-
-    def _process_split_stream(self, flv_header: FlvHeader) -> None:
-        self._complete_file()
-
-        first_data_tag: FlvTag
-
-        if self._stream_cutter.is_cutting():
-            assert self._stream_cutter.last_keyframe_tag is not None
-            last_keyframe_tag = self._stream_cutter.last_keyframe_tag
-            original_ts = last_keyframe_tag.timestamp - self._delta
-            first_data_tag = last_keyframe_tag.evolve(timestamp=original_ts)
-        elif (
-            not self._disable_limit and (
-                self._limit_checker.is_filesize_over_limit() or
-                self._limit_checker.is_duration_over_limit()
-            )
-        ):
-            assert self._limit_checker.last_keyframe_tag is not None
-            last_keyframe_tag = self._limit_checker.last_keyframe_tag
-            original_ts = last_keyframe_tag.timestamp - self._delta
-            first_data_tag = last_keyframe_tag.evolve(timestamp=original_ts)
-        else:
-            first_data_tag = self._read_first_data_tag()
-
-        try:
-            self._process_initial_stream(flv_header, first_data_tag)
-        except (
-            AudioParametersChanged, VideoParametersChanged,
-            FileSizeOverLimit, DurationOverLimit, CutStream,
-        ):
-            self._process_split_stream(flv_header)
-
-    def _transfer_meta_tags(self) -> None:
-        logger.debug('Transfering meta tags...')
-
-        if self._parameters_checker.last_metadata_tag is None:
-            raise FlvStreamCorruptedError('No metadata tag in the stream')
-        if self._parameters_checker.last_video_header_tag is None:
-            raise FlvStreamCorruptedError('No video header tag in the stream')
-        if self._has_audio:
-            if self._parameters_checker.last_audio_header_tag is None:
-                raise FlvStreamCorruptedError(
-                    'No audio header tag in the stream'
-                )
-        metadata_tag = self._parameters_checker.last_metadata_tag
-        video_header_tag = self._parameters_checker.last_video_header_tag
-        audio_header_tag = self._parameters_checker.last_audio_header_tag
-
-        offset = self._out_file.tell()
-        self._metadata_tag = self._enrich_metadata(metadata_tag, offset)
-
-        offset_delta = self._metadata_tag.tag_size - metadata_tag.tag_size
-        self._update_injected_metadata(offset_delta)
-
-        self._write_tag(
-            self._correct_ts(self._metadata_tag, -self._metadata_tag.timestamp)
-        )
-        self._write_tag(
-            self._correct_ts(video_header_tag, -video_header_tag.timestamp)
-        )
-        if audio_header_tag is not None:
-            self._write_tag(
-                self._correct_ts(audio_header_tag, -audio_header_tag.timestamp)
-            )
-
-        logger.debug('Meta tags have been transfered')
-
-    def _update_stream_profile(
-        self, flv_header: FlvHeader, first_data_tag: FlvTag
-    ) -> None:
-        from ..core.stream_analyzer import ffprobe
-
-        if self._parameters_checker.last_metadata_tag is None:
-            return
-        if self._parameters_checker.last_video_header_tag is None:
-            return
-
-        bytes_io = io.BytesIO()
-        writer = FlvWriter(bytes_io)
-        writer.write_header(flv_header)
-        writer.write_tag(self._parameters_checker.last_metadata_tag)
-        writer.write_tag(self._parameters_checker.last_video_header_tag)
-        if self._parameters_checker.last_audio_header_tag is not None:
-            writer.write_tag(self._parameters_checker.last_audio_header_tag)
-        writer.write_tag(first_data_tag)
-
-        def on_next(profile: StreamProfile) -> None:
-            self._stream_profile_updates.on_next(profile)
-
-        def on_error(e: Exception) -> None:
-            logger.warning(f'Failed to analyse stream: {repr(e)}')
-
-        ffprobe(bytes_io.getvalue()).subscribe(on_next, on_error)
-
-    def _transfer_first_data_tag(self, tag: FlvTag) -> None:
-        logger.debug(f'Transfer the first data tag: {tag}')
-        self._delta = -tag.timestamp
-        self._transfer_tags([tag])
-        logger.debug('The first data tag has been transfered')
-
-    def _read_tags_for_deduplication(
-        self
-    ) -> Tuple[List[FlvTag], Optional[Exception]]:
-        logger.debug('Reading tags for tag deduplication...')
-        tags: List[FlvTag] = []
-        try:
-            for tag in read_tags_in_duration(
-                self._in_reader, self._MAX_DURATION
-            ):
-                tags.append(tag)
-        except Exception as exc:
-            logger.debug(f'Failed to read data, due to: {repr(exc)}')
-            return tags, exc
-        else:
-            return tags, None
-        finally:
-            logger.debug('Read {} tags, total size: {}'.format(
-                len(tags), sum(t.tag_size for t in tags)
-            ))
-
-    def _transfer_tags_until_complete(self) -> None:
-        logger.debug('Transfering tags until complete...')
-        self._transfer_tags(self._read_tags_from_in_stream())
-
-    def _read_tags_from_in_stream(self) -> Iterator[FlvTag]:
-        while not self._cancelled:
-            try:
-                tag = self._in_reader.read_tag()
-                self._parameters_checker.check_tag(tag)
-                yield tag
-            except EOFError:
-                logger.debug('The input stream exhausted')
-                break
-            except AudioParametersChanged:
-                if self._analyse_data:
-                    logger.warning('Audio parameters changed at {}'.format(
-                        format_timestamp(self._data_analyser.last_timestamp),
-                    ))
-                yield tag
-            except VideoParametersChanged:
-                if self._analyse_data:
-                    logger.warning('Video parameters changed at {}'.format(
-                        format_timestamp(self._data_analyser.last_timestamp),
-                    ))
-                raise
-            except Exception as e:
-                logger.debug(f'Failed to read data, due to: {repr(e)}')
-                raise
-        else:
-            logger.debug('Cancelled reading tags from the input stream')
-
-    def _transfer_tags(self, tags: Iterable[FlvTag]) -> None:
-        logger.debug(f'Transfering tags... timestamp delta: {self._delta}')
-
-        try:
-            count: int = 0
-            for tag in tags:
-                self._ensure_ts_correct(tag)
-                self._write_tag(self._correct_ts(tag, self._delta))
-                count += 1
-        finally:
-            logger.debug(f'{count} tags have been transfered')
-
-    def _add_join_point(
-        self, offset: int, timestamp: int, seamless: bool
-    ) -> None:
-        join_point = JoinPoint(offset, timestamp, seamless)
-        self._join_points.append(join_point)
-        logger.debug(f'{repr(join_point)}; {join_point}')
-
-    def _find_last_duplicated_tag(self, tags: List[FlvTag]) -> int:
-        logger.debug('Finding duplicated tags...')
-
-        last_out_tag = self._last_tags[0]
-        logger.debug(f'The last output tag is {last_out_tag}')
-
-        for idx, tag in enumerate(tags):
-            if not tag.is_the_same_as(last_out_tag):
-                continue
-
-            if not all(
-                map(
-                    lambda t: t[0].is_the_same_as(t[1]),
-                    zip(reversed(tags[:idx]), self._last_tags[1:])
-                )
-            ):
-                continue
-
-            logger.debug(f'The last duplicated tag found at {idx} is {tag}')
-            return idx
-
-        logger.debug('No duplicated tags found')
-        return -1
-
-    def _read_header(self) -> FlvHeader:
-        try:
-            return self._in_reader.read_header()
-        except Exception as exc:
-            raise FlvStreamCorruptedError(repr(exc))
-
-    def _read_first_data_tag(self) -> Union[AudioTag, VideoTag]:
-        for tag in self._read_tags_from_in_stream():
-            if is_data_tag(tag):
-                return tag
-        raise FlvStreamCorruptedError('No data tag found in the stream!')
-
-    def _read_tags(self, count: int) -> Iterator[FlvTag]:
-        assert count > 0, 'count must greater than 0'
-        for c, tag in enumerate(self._read_tags_from_in_stream(), start=1):
-            yield tag
-            if c >= count:
-                break
-
-    def _write_header(self, header: FlvHeader) -> None:
-        try:
-            size = self._out_writer.write_header(header)
-            logger.debug('The flv header has been copied')
-        except Exception as exc:
-            logger.debug(f'Failed to write data, due to: {repr(exc)}')
-            raise
-
-        if not self._disable_limit:
-            self._limit_checker.check_header(header)
-        if self._analyse_data:
-            self._data_analyser.analyse_header(header)
-        self._size_updates.on_next(size)
-
-    def _write_tag(self, tag: FlvTag) -> None:
-        offset = self._out_file.tell()
-        tag = tag.evolve(offset=offset)
-
-        try:
-            size = self._out_writer.write_tag(tag)
-        except Exception as exc:
-            logger.debug(f'Failed to write data, due to: {repr(exc)}')
-            raise
-        else:
-            self._last_tags.insert(0, tag)
-            if len(self._last_tags) > self._TAG_SEQUENCE_COUNT:
-                self._last_tags.pop()
-
-        self._stream_cutter.check_tag(tag)
-        if not self._disable_limit:
-            self._limit_checker.check_tag(tag)
-        if self._analyse_data:
-            self._data_analyser.analyse_tag(tag)
-
-        self._size_updates.on_next(size)
-        self._time_updates.on_next(tag.timestamp)
-
-    def _ensure_header_correct(self, header: FlvHeader) -> FlvHeader:
-        header.set_video_flag(
-            self._parameters_checker.last_video_header_tag is not None
-        )
-        header.set_audio_flag(
-            self._parameters_checker.last_audio_header_tag is not None
-        )
-        return header
-
-    def _ensure_ts_correct(self, tag: FlvTag) -> None:
-        if not is_audio_data_tag(tag) or not is_video_data_tag(tag):
-            return
-        if tag.timestamp + self._delta < 0:
-            self._delta = -tag.timestamp
-            logger.warning('Incorrect timestamp: {}, new delta: {}'.format(
-                tag, self._delta
-            ))
-
-    def _correct_ts(self, tag: FlvTag, delta: int) -> FlvTag:
-        if delta == 0 and tag.timestamp >= 0:
-            return tag
-        return tag.evolve(timestamp=max(0, tag.timestamp + delta))
-
-    def _calc_delta_duplicated(self, last_duplicated_tag: FlvTag) -> int:
-        return self._last_tags[0].timestamp - last_duplicated_tag.timestamp
-
-    def _calc_delta_no_duplicated(self, first_data_tag: FlvTag) -> int:
-        return (
-            self._last_tags[0].timestamp - first_data_tag.timestamp +
-            self._in_reader.calc_interval(first_data_tag)
-        )
-
-    def _enrich_metadata(
-        self, old_metadata_tag: ScriptTag, offset: int
-    ) -> ScriptTag:
-        # ensure nesessary properties exists in the metadata and init them.
-        metadata = parse_metadata(old_metadata_tag)
-        self._metadata.update({
-            'duration': 0.0,
-            'filesize': 0.0,
-            'framerate': metadata.get('framerate', metadata.get('fps', 0.0)),
-        })
-        # merge the metadata into the metadata tag
-        return enrich_metadata(old_metadata_tag, self._metadata, offset)
-
-    def _update_injected_metadata(self, offset_delta: int) -> None:
-        updated = False
-
-        if (keyframes := self._metadata.get('keyframes')):
-            keyframes['filepositions'] = list(
-                map(lambda p: p + offset_delta, keyframes['filepositions'])
-            )
-            if 'lastkeyframelocation' in self._metadata:
-                self._metadata['lastkeyframelocation'] = \
-                    keyframes['filepositions'][-1]
-            updated = True
-
-        if (join_points := self._metadata.get('joinpoints')):
-            join_points = cast(List[JoinPointData], join_points)
-            self._metadata['joinpoints'] = list(
-                {**p, 'offset': p['offset'] + offset_delta}
-                for p in join_points
-            )
-            updated = True
-
-        if updated:
-            self._metadata_tag = \
-                update_metadata(self._metadata_tag, self._metadata)
-
-    def _update_metadata_tag(self) -> None:
-        last_tag = self._last_tags[0]
-        duration = last_tag.timestamp / 1000
-        filesize = float(last_tag.next_tag_offset)
-        updates = {
-            'duration': duration,
-            'filesize': filesize,
-        }
-        if self._analyse_data:
-            updates.update({
-                'framerate': self._data_analyser.calc_frame_rate(),
-            })
-        self._metadata_tag = update_metadata(self._metadata_tag, updates)
-        self._out_file.seek(self._metadata_tag.offset)
-        self._out_writer.write_tag(self._metadata_tag)
-        logger.debug('The metadata tag has been updated')
-
-    def _save_extra_metadata(self) -> None:
-        if self._analyse_data:
-            metadata = attr.asdict(
-                self._data_analyser.make_metadata(),
-                filter=lambda a, v: v is not None,
-            )
-        else:
-            metadata = {}
-
-        metadata['joinpoints'] = list(
-            map(lambda p: p.to_metadata_value(), self._join_points)
-        )
-
-        assert self._file_manager.curr_path is not None
-        path = extra_metadata_path(self._file_manager.curr_path)
-        with open(path, 'wt', encoding='utf8') as file:
-            json.dump(metadata, file)
-
-        logger.debug('The extra metadata has been saved')
-
-
-@attr.s(auto_attribs=True, slots=True, frozen=True)
-class JoinPoint:
-    offset: int
-    timestamp: int  # milliseconds
-    seamless: bool
-
-    @classmethod
-    def from_metadata_value(cls, value: JoinPointData) -> JoinPoint:
-        return cls(
-            offset=int(value['offset']),
-            timestamp=int(value['timestamp']),
-            seamless=value['seamless'],
-        )
-
-    def to_metadata_value(self) -> JoinPointData:
-        return dict(
-            offset=float(self.offset),
-            timestamp=float(self.timestamp),
-            seamless=self.seamless,
-        )
-
-    def __str__(self) -> str:
-        return 'offset: {}, timestamp: {}, seamless: {}'.format(
-            format_offest(self.offset),
-            format_timestamp(self.timestamp),
-            'yes' if self.seamless else 'no',
-        )
-
-
-class JoinPointData(TypedDict):
-    offset: float
-    timestamp: float
-    seamless: bool
-
-
-class OutputFileManager(Protocol):
-    @property
-    def curr_path(self) -> Optional[str]:
-        ...
-
-    @property
-    def curr_file(self) -> Optional[BinaryIO]:
-        ...
-
-    def create_file(self) -> BinaryIO:
-        ...
-
-    def close_file(self) -> None:
-        ...
-
-
-class BaseOutputFileManager(ABC):
-    def __init__(self, buffer_size: Optional[int] = None) -> None:
-        super().__init__()
-        self.buffer_size = buffer_size or io.DEFAULT_BUFFER_SIZE  # bytes
-        self._paths: List[str] = []
-        self._curr_path: Optional[str] = None
-        self._curr_file: Optional[BinaryIO] = None
-
-    @property
-    def curr_path(self) -> Optional[str]:
-        return self._curr_path
-
-    @property
-    def curr_file(self) -> Optional[BinaryIO]:
-        return self._curr_file
-
-    def has_file(self) -> bool:
-        return len(self._paths) > 1
-
-    def get_files(self) -> Iterator[str]:
-        for file in self._paths:
-            yield file
-
-    def clear_files(self) -> None:
-        self._paths.clear()
-
-    def create_file(self) -> BinaryIO:
-        assert self._curr_file is None
-        path = self._make_path()
-        file = open(path, mode='w+b', buffering=self.buffer_size)
-        self._paths.append(path)
-        self._curr_path = path
-        self._curr_file = file
-        return file
-
-    def close_file(self) -> None:
-        assert self._curr_file is not None
-        self._curr_file.close()
-        self._curr_path = None
-        self._curr_file = None
-
-    @abstractmethod
-    def _make_path(self) -> str:
-        ...
-
-
-class RobustFlvReader(FlvReader):
-    def read_tag(self, *, no_body: bool = False) -> FlvTag:
-        count = 0
-        while True:
-            try:
-                tag = super().read_tag(no_body=no_body)
-            except FlvTagError as e:
-                logger.warning(f'Invalid tag: {repr(e)}')
-                self._parser.parse_previous_tag_size()
-                count += 1
-                if count > 3:
-                    raise
-            else:
-                count = 0
-                return tag
-
-
-class FlvReaderWithTimestampFix(RobustFlvReader):
-    def __init__(self, stream: RandomIO) -> None:
-        super().__init__(stream)
-        self._last_tag: Optional[FlvTag] = None
-        self._last_video_tag: Optional[VideoTag] = None
-        self._last_audio_tag: Optional[AudioTag] = None
-        self._delta = 0
-        # 15 is probably the minimal frame rate
-        self._frame_rate = 15.0
-        self._video_frame_interval = math.ceil(1000 / self._frame_rate)
-        # AAC SoundRate always is 44 KHz
-        self._sound_sample_interval = math.ceil(1000 / 44)
-
-    @property
-    def frame_rate(self) -> float:
-        return self._frame_rate
-
-    @property
-    def sound_sample_interval(self) -> int:
-        return self._sound_sample_interval
-
-    @property
-    def video_frame_interval(self) -> int:
-        return self._video_frame_interval
-
-    def read_tag(self, *, no_body: bool = False) -> FlvTag:
-        while True:
-            tag = super().read_tag(no_body=no_body)
-
-            if self._last_tag is None:
-                if is_video_tag(tag) or is_audio_tag(tag):
-                    self._update_last_tags(tag)
-                elif is_metadata_tag(tag):
-                    self._update_parameters(tag)
-                return tag
-
-            if not is_video_tag(tag) and not is_audio_tag(tag):
-                return tag
-
-            if self._is_ts_rebounded(tag):
-                self._update_delta(tag)
-                logger.warning(
-                    f'Timestamp rebounded, updated delta: {self._delta}\n'
-                    f'last video tag: {self._last_video_tag}\n'
-                    f'last audio tag: {self._last_audio_tag}\n'
-                    f'current tag: {tag}'
-                )
-
-            if self._is_ts_jumped(tag):
-                self._update_delta(tag)
-                logger.warning(
-                    f'Timestamp jumped, updated delta: {self._delta}\n'
-                    f'last tag: {self._last_tag}\n'
-                    f'current tag: {tag}'
-                )
-
-            self._update_last_tags(tag)
-
-            return self._correct_ts(tag)
-
-    def rread_tag(self, *, no_body: bool = False) -> FlvTag:
-        raise NotImplementedError()
-
-    def read_body(self, tag: FlvTag) -> bytes:
-        raise NotImplementedError()
-
-    def calc_interval(self, tag: FlvTag) -> int:
-        if is_audio_tag(tag):
-            return self._sound_sample_interval
-        elif is_video_tag(tag):
-            return self._video_frame_interval
-        else:
-            logger.warning(f'Unexpected tag type: {tag}')
-            return min(self._sound_sample_interval, self._video_frame_interval)
-
-    def _is_ts_rebounded(self, tag: FlvTag) -> bool:
-        if is_video_tag(tag):
-            if self._last_video_tag is None:
-                return False
-            return tag.timestamp <= self._last_video_tag.timestamp
-        elif is_audio_tag(tag):
-            if self._last_audio_tag is None:
-                return False
-            return tag.timestamp <= self._last_audio_tag.timestamp
-        else:
-            return False
-
-    def _is_ts_jumped(self, tag: FlvTag) -> bool:
-        assert self._last_tag is not None
-        return (
-            tag.timestamp - self._last_tag.timestamp >
-            max(self._sound_sample_interval, self._video_frame_interval)
-        )
-
-    def _update_last_tags(self, tag: FlvTag) -> None:
-        self._last_tag = tag
-        if is_video_tag(tag):
-            self._last_video_tag = tag
-        elif is_audio_tag(tag):
-            self._last_audio_tag = tag
-
-    def _update_parameters(self, tag: ScriptTag) -> None:
-        metadata = parse_metadata(tag)
-        frame_rate = metadata.get('fps') or metadata.get('framerate')
-
-        if not frame_rate:
-            return
-
-        self._frame_rate = frame_rate
-        self._video_frame_interval = math.ceil(1000 / frame_rate)
-
-        logger.debug('frame rate: {}, video frame interval: {}'.format(
-            frame_rate, self._video_frame_interval
-        ))
-
-    def _update_delta(self, tag: FlvTag) -> None:
-        assert self._last_tag is not None
-        self._delta = (
-            self._last_tag.timestamp + self._delta - tag.timestamp +
-            self.calc_interval(tag)
-        )
-
-    def _correct_ts(self, tag: FlvTag) -> FlvTag:
-        if self._delta == 0:
-            return tag
-        return tag.evolve(timestamp=tag.timestamp + self._delta)

src/blrec/notification/__init__.py

@@ -3,13 +3,17 @@ from .notifiers import (
     MessageNotifier,
     EmailNotifier,
     ServerchanNotifier,
+    PushdeerNotifier,
     PushplusNotifier,
+    TelegramNotifier,
 )
 from .providers import (
     MessagingProvider,
     EmailService,
     Serverchan,
+    Pushdeer,
     Pushplus,
+    Telegram,
 )
 
 
@@ -17,11 +21,15 @@ __all__ = (
     'MessagingProvider',
     'EmailService',
     'Serverchan',
+    'Pushdeer',
     'Pushplus',
+    'Telegram',
 
     'Notifier',
     'MessageNotifier',
     'EmailNotifier',
     'ServerchanNotifier',
+    'PushdeerNotifier',
     'PushplusNotifier',
+    'TelegramNotifier',
 )

src/blrec/notification/notifiers.py

@@ -13,7 +13,9 @@ from .providers import (
     EmailService,
     MessagingProvider,
     Serverchan,
+    Pushdeer,
     Pushplus,
+    Telegram,
 )
 from .message import (
     make_live_info_content,
@@ -36,7 +38,9 @@ __all__ = (
     'MessageNotifier',
     'EmailNotifier',
     'ServerchanNotifier',
-    'PushplusNotifier'
+    'PushdeerNotifier',
+    'PushplusNotifier',
+    'TelegramNotifier',
 )
 
 
@@ -188,6 +192,18 @@ class ServerchanNotifier(MessageNotifier):
         logger.debug('Disabled Serverchan notifier')
 
 
+class PushdeerNotifier(MessageNotifier):
+    provider = Pushdeer.get_instance()
+
+    def _do_enable(self) -> None:
+        super()._do_enable()
+        logger.debug('Enabled Pushdeer notifier')
+
+    def _do_disable(self) -> None:
+        super()._do_disable()
+        logger.debug('Disabled Pushdeer notifier')
+
+
 class PushplusNotifier(MessageNotifier):
     provider = Pushplus.get_instance()
 
@@ -198,3 +214,15 @@ class PushplusNotifier(MessageNotifier):
     def _do_disable(self) -> None:
         super()._do_disable()
         logger.debug('Disabled Pushplus notifier')
+
+
+class TelegramNotifier(MessageNotifier):
+    provider = Telegram.get_instance()
+
+    def _do_enable(self) -> None:
+        super()._do_enable()
+        logger.debug('Enabled Telegram notifier')
+
+    def _do_disable(self) -> None:
+        super()._do_disable()
+        logger.debug('Disabled Telegram notifier')

src/blrec/notification/providers.py

@@ -1,18 +1,25 @@
-import ssl
-import logging
 import asyncio
+import logging
 import smtplib
+import ssl
 from abc import ABC, abstractmethod
-from typing import TypedDict, cast, Literal
 from email.message import EmailMessage
 from http.client import HTTPException
+from typing import Final, Literal, TypedDict, Dict, Any, cast
+from urllib.parse import urljoin
 
 import aiohttp
 
 from ..utils.patterns import Singleton
 
-
-__all__ = 'MessagingProvider', 'EmailService', 'Serverchan', 'Pushplus'
+__all__ = (
+    'MessagingProvider',
+    'EmailService',
+    'Serverchan',
+    'Pushdeer',
+    'Pushplus',
+    'Telegram',
+)
 
 
 logger = logging.getLogger(__name__)
@@ -107,6 +114,44 @@ class Serverchan(MessagingProvider):
                 pass
 
 
+class PushdeerResponse(TypedDict):
+    code: int
+    content: str
+    error: str
+
+
+class Pushdeer(MessagingProvider):
+    _server: Final = 'https://api2.pushdeer.com'
+    _endpoint: Final = '/message/push'
+
+    def __init__(self, server: str = '', pushkey: str = '') -> None:
+        super().__init__()
+        self.server = server
+        self.pushkey = pushkey
+
+    async def send_message(self, title: str, content: str) -> None:
+        self._check_parameters()
+        await self._post_message(title, content)
+
+    def _check_parameters(self) -> None:
+        if not self.pushkey:
+            raise ValueError('No pushkey supplied')
+
+    async def _post_message(self, title: str, content: str) -> None:
+        url = urljoin(self.server or self._server, self._endpoint)
+        payload = {
+            'pushkey': self.pushkey,
+            'text': title,
+            'desp': content,
+            'type': 'text',
+        }
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            async with session.post(url, json=payload) as res:
+                response = cast(PushdeerResponse, await res.json())
+                if response['code'] != 0:
+                    raise HTTPException(response['code'], response['error'])
+
+
 class PushplusResponse(TypedDict):
     code: int
     msg: str
@@ -143,3 +188,42 @@ class Pushplus(MessagingProvider):
                 response = cast(PushplusResponse, await res.json())
                 if response['code'] != 200:
                     raise HTTPException(response['code'], response['msg'])
+
+
+class TelegramResponse(TypedDict):
+    ok: bool
+    result: Dict[str, Any]
+
+
+class Telegram(MessagingProvider):
+    def __init__(self, token: str = '', chatid: str = '') -> None:
+        super().__init__()
+        self.token = token
+        self.chatid = chatid
+
+    async def send_message(self, title: str, content: str) -> None:
+        self._check_parameters()
+        await self._post_message(title, content)
+
+    def _check_parameters(self) -> None:
+        if not self.token:
+            raise ValueError('No token supplied')
+        if not self.chatid:
+            raise ValueError('No chatid supplied')
+
+    async def _post_message(self, title: str, content: str) -> None:
+        url = f'https://api.telegram.org/bot{self.token}/sendMessage'
+        payload = {
+            'chat_id': self.chatid,
+            'text': title + '\n' + content,
+            'parse_mode': 'HTML',
+        }
+
+        async with aiohttp.ClientSession(raise_for_status=True) as session:
+            async with session.post(url, json=payload) as res:
+                response = cast(TelegramResponse, await res.json())
+                if not response['ok']:
+                    raise HTTPException(
+                        response['result']['error_code'],
+                        response['result']['description'],
+                    )

src/blrec/postprocess/ffmpeg_metadata.py

@@ -1,13 +1,12 @@
-import logging
 import json
+import logging
 from typing import Iterable, cast
 
 import aiofiles
 
-from .helpers import get_metadata, get_extra_metadata
-from ..flv.stream_processor import JoinPoint
 from ..flv.helpers import make_comment_for_joinpoints
-
+from ..flv.operators import JoinPoint
+from .helpers import get_extra_metadata, get_metadata
 
 logger = logging.getLogger(__name__)
 
@@ -27,7 +26,7 @@ async def _make_metadata_content(flv_path: str) -> str:
     comment = cast(str, metadata.get('Comment', ''))
     chapters = ''
 
-    if (join_points := extra_metadata.get('joinpoints')):
+    if join_points := extra_metadata.get('joinpoints'):
         join_points = list(map(JoinPoint.from_metadata_value, join_points))
         comment += '\n\n' + make_comment_for_joinpoints(join_points)
         duration = int(cast(float, metadata['duration']) * 1000)
@@ -49,9 +48,7 @@ Comment={comment}
 """
 
 
-def _make_chapters(
-    join_points: Iterable[JoinPoint], duration: int
-) -> str:
+def _make_chapters(join_points: Iterable[JoinPoint], duration: int) -> str:
     join_points = filter(lambda p: not p.seamless, join_points)
     timestamps = list(map(lambda p: p.timestamp, join_points))
     if not timestamps:

src/blrec/postprocess/postprocessor.py

@@ -1,28 +1,27 @@
 from __future__ import annotations
+
 import asyncio
 import logging
-from pathlib import PurePath
 from contextlib import suppress
+from pathlib import PurePath
 from typing import Any, Awaitable, Dict, Iterator, List, Optional, Union
 
-from rx.core.typing import Scheduler
-from rx.scheduler.threadpoolscheduler import ThreadPoolScheduler
+from reactivex.scheduler import ThreadPoolScheduler
 
-from .models import PostprocessorStatus, DeleteStrategy
-from .typing import Progress
-from .remuxer import remux_video, RemuxProgress, RemuxResult
-from .helpers import discard_file, get_extra_metadata
-from .ffmpeg_metadata import make_metadata_file
-from ..event.event_emitter import EventListener, EventEmitter
 from ..bili.live import Live
 from ..core import Recorder, RecorderEventListener
-from ..exception import submit_exception
-from ..utils.mixins import AsyncStoppableMixin, AsyncCooperationMixin
-from ..path import extra_metadata_path
-from ..flv.metadata_injector import inject_metadata, InjectProgress
+from ..event.event_emitter import EventEmitter, EventListener
+from ..exception import exception_callback, submit_exception
 from ..flv.helpers import is_valid_flv_file
+from ..flv.metadata_injection import InjectingProgress, inject_metadata
 from ..logging.room_id import aio_task_with_room_id
-
+from ..path import extra_metadata_path
+from ..utils.mixins import AsyncCooperationMixin, AsyncStoppableMixin, SupportDebugMixin
+from .ffmpeg_metadata import make_metadata_file
+from .helpers import discard_file, get_extra_metadata
+from .models import DeleteStrategy, PostprocessorStatus
+from .remux import RemuxingProgress, RemuxingResult, remux_video
+from .typing import Progress
 
 __all__ = (
     'Postprocessor',
@@ -47,6 +46,7 @@ class Postprocessor(
     RecorderEventListener,
     AsyncStoppableMixin,
     AsyncCooperationMixin,
+    SupportDebugMixin,
 ):
     def __init__(
         self,
@@ -58,6 +58,7 @@ class Postprocessor(
         delete_source: DeleteStrategy = DeleteStrategy.AUTO,
     ) -> None:
         super().__init__()
+        self._init_for_debug(live.room_id)
 
         self._live = live
         self._recorder = recorder
@@ -94,14 +95,10 @@ class Postprocessor(
         # clear completed files of previous recording
         self._completed_files.clear()
 
-    async def on_video_file_completed(
-        self, recorder: Recorder, path: str
-    ) -> None:
+    async def on_video_file_completed(self, recorder: Recorder, path: str) -> None:
         self._queue.put_nowait(path)
 
-    async def on_danmaku_file_completed(
-        self, recorder: Recorder, path: str
-    ) -> None:
+    async def on_danmaku_file_completed(self, recorder: Recorder, path: str) -> None:
         self._completed_files.append(path)
 
     async def _do_start(self) -> None:
@@ -110,6 +107,7 @@ class Postprocessor(
         self._queue: asyncio.Queue[str] = asyncio.Queue()
         self._scheduler = ThreadPoolScheduler()
         self._task = asyncio.create_task(self._worker())
+        self._task.add_done_callback(exception_callback)
 
         logger.debug('Started postprocessor')
 
@@ -150,12 +148,11 @@ class Postprocessor(
                 else:
                     result_path = video_path
 
-                await discard_file(extra_metadata_path(video_path), 'DEBUG')
+                if not self._debug:
+                    await discard_file(extra_metadata_path(video_path), 'DEBUG')
 
                 self._completed_files.append(result_path)
-                await self._emit(
-                    'video_postprocessing_completed', self, result_path,
-                )
+                await self._emit('video_postprocessing_completed', self, result_path)
             except Exception as exc:
                 submit_exception(exc)
             finally:
@@ -164,9 +161,13 @@ class Postprocessor(
     async def _inject_extra_metadata(self, path: str) -> str:
         try:
             metadata = await get_extra_metadata(path)
-            await self._inject_metadata(path, metadata, self._scheduler)
+            logger.info(f"Injecting metadata for '{path}' ...")
+            await self._inject_metadata(path, metadata)
         except Exception as e:
             logger.error(f"Failed to inject metadata for '{path}': {repr(e)}")
+            submit_exception(e)
+        else:
+            logger.info(f"Successfully injected metadata for '{path}'")
         return path
 
     async def _remux_flv_to_mp4(self, in_path: str) -> str:
@@ -174,12 +175,10 @@ class Postprocessor(
         logger.info(f"Remuxing '{in_path}' to '{out_path}' ...")
 
         metadata_path = await make_metadata_file(in_path)
-        remux_result = await self._remux_video(
-            in_path, out_path, metadata_path, self._scheduler
-        )
+        remux_result = await self._remux_video(in_path, out_path, metadata_path)
 
         if remux_result.is_successful():
-            logger.info(f"Successfully remux '{in_path}' to '{out_path}'")
+            logger.info(f"Successfully remuxed '{in_path}' to '{out_path}'")
             result_path = out_path
         elif remux_result.is_warned():
             logger.warning('Remuxing done, but ran into problems.')
@@ -192,64 +191,54 @@ class Postprocessor(
 
         logger.debug(f'ffmpeg output:\n{remux_result.output}')
 
-        await discard_file(metadata_path, 'DEBUG')
+        if not self._debug:
+            await discard_file(metadata_path, 'DEBUG')
         if self._should_delete_source_files(remux_result):
             await discard_file(in_path)
 
         return result_path
 
-    def _inject_metadata(
-        self,
-        path: str,
-        metadata: Dict[str, Any],
-        scheduler: Scheduler,
-    ) -> Awaitable[None]:
+    def _inject_metadata(self, path: str, metadata: Dict[str, Any]) -> Awaitable[None]:
         future: asyncio.Future[None] = asyncio.Future()
         self._postprocessing_path = path
 
-        def on_next(value: InjectProgress) -> None:
+        def on_next(value: InjectingProgress) -> None:
             self._postprocessing_progress = value
 
-        inject_metadata(
-            path,
-            metadata,
-            report_progress=True,
-            room_id=self._live.room_id,
-        ).subscribe(
-            on_next,
-            lambda e: future.set_exception(e),
-            lambda: future.set_result(None),
-            scheduler=scheduler,
+        subscription = inject_metadata(path, metadata, show_progress=True).subscribe(
+            on_next=on_next,
+            on_error=lambda e: future.set_exception(e),
+            on_completed=lambda: future.set_result(None),
+            scheduler=self._scheduler,
         )
+        future.add_done_callback(lambda f: subscription.dispose())
 
         return future
 
     def _remux_video(
-        self,
-        in_path: str,
-        out_path: str,
-        metadata_path: str,
-        scheduler: Scheduler,
-    ) -> Awaitable[RemuxResult]:
-        future: asyncio.Future[RemuxResult] = asyncio.Future()
+        self, in_path: str, out_path: str, metadata_path: str
+    ) -> Awaitable[RemuxingResult]:
+        future: asyncio.Future[RemuxingResult] = asyncio.Future()
         self._postprocessing_path = in_path
 
-        def on_next(value: Union[RemuxProgress, RemuxResult]) -> None:
-            if isinstance(value, RemuxProgress):
+        def on_next(value: Union[RemuxingProgress, RemuxingResult]) -> None:
+            if isinstance(value, RemuxingProgress):
                 self._postprocessing_progress = value
-            elif isinstance(value, RemuxResult):
+            elif isinstance(value, RemuxingResult):
                 future.set_result(value)
 
-        remux_video(
+        subscription = remux_video(
             in_path,
             out_path,
             metadata_path,
-            report_progress=True,
+            show_progress=True,
+            remove_filler_data=True,
         ).subscribe(
-            on_next,
-            lambda e: future.set_exception(e),
-            scheduler=scheduler,
+            on_next=on_next,
+            on_error=lambda e: future.set_exception(e),
+            scheduler=self._scheduler,
         )
+        future.add_done_callback(lambda f: subscription.dispose())
 
         return future
 
@@ -257,9 +246,7 @@ class Postprocessor(
         loop = asyncio.get_running_loop()
         return await loop.run_in_executor(None, is_valid_flv_file, video_path)
 
-    def _should_delete_source_files(
-        self, remux_result: RemuxResult
-    ) -> bool:
+    def _should_delete_source_files(self, remux_result: RemuxingResult) -> bool:
         if self.delete_source == DeleteStrategy.AUTO:
             if not remux_result.is_failed():
                 return True

src/blrec/postprocess/remux.py

@@ -0,0 +1,158 @@
+import os
+import re
+import shlex
+from subprocess import PIPE, Popen
+from typing import Any, Final, List, Optional, Union
+
+import attr
+from reactivex import Observable, abc, create
+from reactivex.disposable import CompositeDisposable, Disposable, SerialDisposable
+from reactivex.scheduler.currentthreadscheduler import CurrentThreadScheduler
+from tqdm import tqdm
+
+__all__ = 'RemuxingResult', 'remux_video'
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True)
+class RemuxingProgress:
+    count: int
+    total: int
+
+
+@attr.s(auto_attribs=True, slots=True, frozen=True)
+class RemuxingResult:
+    return_code: int
+    output: str
+
+    def is_done(self) -> bool:
+        return self.return_code == 0
+
+    def is_successful(self) -> bool:
+        return self.is_done() and not self.may_timestamps_incorrect()
+
+    def is_warned(self) -> bool:
+        return self.is_done() and self.may_timestamps_incorrect()
+
+    def is_failed(self) -> bool:
+        return not self.is_done()
+
+    def may_timestamps_incorrect(self) -> bool:
+        return 'Non-monotonous DTS in output stream' in self.output
+
+
+def remux_video(
+    in_path: str,
+    out_path: str,
+    metadata_path: Optional[str] = None,
+    *,
+    show_progress: bool = False,
+    remove_filler_data: bool = False,
+) -> Observable[Union[RemuxingProgress, RemuxingResult]]:
+    SIZE_PATTERN: Final = re.compile(r'size=\s*(?P<number>\d+)(?P<unit>[a-zA-Z]?B)')
+    filesize = os.path.getsize(in_path)
+    filename = os.path.basename(in_path)
+
+    def parse_size(line: str) -> int:
+        match = SIZE_PATTERN.search(line)
+        assert match is not None
+        result = match.groupdict()
+
+        unit = result['unit']
+        number = int(result['number'])
+
+        if unit == 'B':
+            size = number
+        elif unit == 'kB':
+            size = 1024 * number
+        elif unit == 'MB':
+            size = 1024**2 * number
+        elif unit == 'GB':
+            size = 1024**3 * number
+        else:
+            raise ValueError(unit)
+
+        return size
+
+    def should_output_line(line: str) -> bool:
+        line = line.strip()
+        return not (line.startswith('frame=') or line.startswith('Press [q]'))
+
+    def subscribe(
+        observer: abc.ObserverBase[Union[RemuxingProgress, RemuxingResult]],
+        scheduler: Optional[abc.SchedulerBase] = None,
+    ) -> abc.DisposableBase:
+        _scheduler = scheduler or CurrentThreadScheduler.singleton()
+
+        disposed = False
+        cancelable = SerialDisposable()
+
+        def action(scheduler: abc.SchedulerBase, state: Optional[Any] = None) -> None:
+            if disposed:
+                return
+
+            with tqdm(
+                desc='Remuxing',
+                total=filesize,
+                unit='B',
+                unit_scale=True,
+                unit_divisor=1024,
+                postfix=filename,
+                disable=not show_progress,
+            ) as pbar:
+                cmd = f'ffmpeg -i "{in_path}"'
+                if metadata_path is not None:
+                    cmd += f' -i "{metadata_path}" -map_metadata 1'
+                cmd += ' -codec copy'
+                if remove_filler_data:
+                    # https://forum.doom9.org/showthread.php?t=152051
+                    # ISO_IEC_14496-10_2020(E)
+                    # Table 7-1 – NAL unit type codes, syntax element categories, and NAL unit type classes  # noqa
+                    # 7.4.2.7 Filler data RBSP semantics
+                    cmd += ' -bsf:v filter_units=remove_types=12'
+                cmd += f' "{out_path}" -y'
+
+                args = shlex.split(cmd)
+                out_lines: List[str] = []
+
+                try:
+                    with Popen(
+                        args, stderr=PIPE, encoding='utf8', errors='backslashreplace'
+                    ) as process:
+                        assert process.stderr is not None
+                        while not disposed:
+                            line = process.stderr.readline()
+                            if not line:
+                                if process.poll() is not None:
+                                    break
+                                else:
+                                    continue
+
+                            if line.startswith('frame='):
+                                size = parse_size(line)
+                                pbar.update(size - pbar.n)
+                                progress = RemuxingProgress(size, filesize)
+                                observer.on_next(progress)
+
+                            if should_output_line(line):
+                                out_lines.append(line)
+
+                        if not disposed and process.returncode == 0:
+                            pbar.update(filesize)
+                            progress = RemuxingProgress(filesize, filesize)
+                            observer.on_next(progress)
+                except Exception as e:
+                    observer.on_error(e)
+                else:
+                    result = RemuxingResult(process.returncode, ''.join(out_lines))
+                    observer.on_next(result)
+                    observer.on_completed()
+
+        cancelable.disposable = _scheduler.schedule(action)
+
+        def dispose() -> None:
+            nonlocal disposed
+            disposed = True
+
+        return CompositeDisposable(cancelable, Disposable(dispose))
+
+    return create(subscribe)