Coverage for downtify/cookie_refresh.py: 85%

119 statements  

« prev     ^ index     » next       coverage.py v7.17.4, created at 2026-11-21 22:18 +0000

1"""Keep the YouTube ``cookies.txt`` signed in without the user's help. 

2 

3Why exported cookies "expire" after a few hours (henriquesebastiao/downtify 

4issue on VPN users re-uploading cookies): Google's session is not one static 

5token. ``__Secure-1PSIDTS`` / ``__Secure-3PSIDTS`` and ``SIDCC`` are rotated 

6by the browser every few minutes (``POST /RotateCookies``) and by the 

7``Set-Cookie`` headers of ordinary page loads. A jar that never learns the 

8new values is eventually refused, and yt-dlp then reports "The provided 

9YouTube account cookies are no longer valid". The more suspicious the IP 

10(VPNs, datacenters), the sooner that happens. 

11 

12Logging in to Google automatically is not an option: Google blocks 

13scripted and headless sign-ins ("This browser or app may not be secure"), 

14asks for 2FA / device prompts, and OAuth for YouTube was shut off for 

15third-party clients. What *is* possible is to behave like the browser 

16tab the cookies came from - so this module, every 

17``DOWNTIFY_COOKIES_REFRESH_MINUTES`` (default 15, ``0`` turns it off): 

18 

191. asks Google to rotate the session cookies (the same undocumented 

20 ``RotateCookies`` call the browser makes - best effort, a failure is 

21 only logged); 

222. loads ``https://www.youtube.com/`` with the jar, which both refreshes 

23 the YouTube cookies and tells whether the session is still signed in 

24 (``"LOGGED_IN":true`` in the page's config); 

253. merges every cookie that changed back into the stored file 

26 (:func:`downtify.cookies.merge_into`). 

27 

28The result is kept as the store's ``health`` (``GET /api/cookies``) so the 

29settings page can say "signed in, checked 3 min ago" or "YouTube signed 

30these cookies out - upload new ones" instead of downloads failing 

31silently. yt-dlp's own warnings feed the same state 

32(:func:`note_ytdlp_message`), so a download that hits a dead session or a 

33bot check shows up there too. 

34""" 

35 

36from __future__ import annotations 

37 

38import asyncio 

39import os 

40import re 

41import threading 

42from datetime import datetime, timezone 

43from typing import Any, Optional 

44 

45import requests 

46from loguru import logger 

47 

48from .cookies import ( 

49 UNKNOWN_HEALTH, 

50 CookiesStore, 

51 changed_cookies, 

52 load_jar, 

53 merge_into, 

54) 

55 

56#: Minutes between two keep-alive runs; ``0`` disables the loop. 

57ENV_INTERVAL = 'DOWNTIFY_COOKIES_REFRESH_MINUTES' 

58DEFAULT_INTERVAL_MINUTES = 15 

59#: Delay before the first run after start, so boot isn't slowed down. 

60_STARTUP_DELAY_SECONDS = 30 

61 

62_TIMEOUT = 20 

63_USER_AGENT = ( 

64 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' 

65 '(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36' 

66) 

67#: Google's own rotation request: a fixed body, the answer is Set-Cookie. 

68_ROTATE_BODY = '[000,"-0000000000000000000"]' 

69#: Cookie names that mean "this jar belongs to a signed-in account". 

70_ACCOUNT_COOKIES = frozenset({ 

71 'SID', 

72 '__Secure-1PSID', 

73 '__Secure-3PSID', 

74 'SAPISID', 

75 'LOGIN_INFO', 

76}) 

77_LOGGED_IN_RE = re.compile(r'"LOGGED_IN"\s*:\s*(true|false)') 

78 

79#: Fragments of yt-dlp messages that mean the session itself is dead. 

80_SIGNED_OUT_FRAGMENTS = ( 

81 'cookies are no longer valid', 

82 'have likely been rotated in the browser', 

83) 

84#: Fragments of yt-dlp messages that mean YouTube wants a signed-in 

85#: (or less suspicious) client: the VPN / datacenter bot wall. 

86_BOT_CHECK_FRAGMENTS = ("confirm you're not a bot", 'confirm you’re not a bot') 

87 

88#: The store :func:`note_ytdlp_message` reports into (one per process). 

89_bound: dict[str, Optional[CookiesStore]] = {'store': None} 

90_run_lock = threading.Lock() 

91 

92 

93def bind_store(store: Optional[CookiesStore]) -> None: 

94 """Make *store* the one :func:`note_ytdlp_message` reports into.""" 

95 

96 _bound['store'] = store 

97 

98 

99def interval_minutes() -> int: 

100 """``DOWNTIFY_COOKIES_REFRESH_MINUTES``, clamped to 0 or 5..1440.""" 

101 

102 raw = os.getenv(ENV_INTERVAL, '').strip() 

103 if not raw: 

104 return DEFAULT_INTERVAL_MINUTES 

105 try: 

106 value = int(raw) 

107 except ValueError: 

108 logger.warning( 

109 '{}={!r} is not a number; using {} minutes', 

110 ENV_INTERVAL, 

111 raw, 

112 DEFAULT_INTERVAL_MINUTES, 

113 ) 

114 return DEFAULT_INTERVAL_MINUTES 

115 if value <= 0: 

116 return 0 

117 return max(5, min(value, 1440)) 

118 

119 

120def _now() -> str: 

121 return datetime.now(timezone.utc).isoformat() 

122 

123 

124def _set_health(store: CookiesStore, **changes: Any) -> None: 

125 previous = store.health.get('state') 

126 store.health = {**store.health, **changes} 

127 state = store.health.get('state') 

128 if state == 'signed_out' and previous != 'signed_out': 

129 logger.warning( 

130 'YouTube cookies are signed out: {}. Upload a fresh ' 

131 'cookies.txt in Settings > YouTube cookies.', 

132 store.health.get('message') or 'session refused', 

133 ) 

134 elif state == 'valid' and previous == 'signed_out': 

135 logger.info('YouTube cookies are signed in again') 

136 

137 

138def note_ytdlp_message(message: str) -> None: 

139 """Record what a yt-dlp warning/error says about the cookies.""" 

140 

141 store = _bound['store'] 

142 if store is None or not message: 

143 return 

144 lowered = message.lower() 

145 if any(fragment in lowered for fragment in _SIGNED_OUT_FRAGMENTS): 

146 _set_health( 

147 store, 

148 state='signed_out', 

149 checked_at=_now(), 

150 message='yt-dlp: YouTube no longer accepts these cookies', 

151 ) 

152 elif any(fragment in lowered for fragment in _BOT_CHECK_FRAGMENTS): 

153 store.health = {**store.health, 'bot_check_at': _now()} 

154 

155 

156def _new_session() -> requests.Session: 

157 """The HTTP session one keep-alive pass runs on (a seam for tests).""" 

158 

159 return requests.Session() 

160 

161 

162def _rotate(session: requests.Session, host: str) -> None: 

163 """Ask ``https://<host>/RotateCookies`` for fresh session cookies.""" 

164 

165 try: 

166 response = session.post( 

167 f'https://{host}/RotateCookies', 

168 data=_ROTATE_BODY, 

169 headers={ 

170 'Content-Type': 'application/json', 

171 'Origin': f'https://{host}', 

172 }, 

173 timeout=_TIMEOUT, 

174 ) 

175 logger.debug( 

176 'Cookie rotation at {}: HTTP {}', host, response.status_code 

177 ) 

178 except requests.RequestException as exc: 

179 logger.debug('Cookie rotation at {} failed: {}', host, exc) 

180 

181 

182def _has_account_cookie(jar: dict, domain: str) -> bool: 

183 return any( 

184 name in _ACCOUNT_COOKIES and cookie_domain.lstrip('.').endswith(domain) 

185 for (cookie_domain, _path, name) in jar 

186 ) 

187 

188 

189def refresh_cookies(store: CookiesStore) -> dict[str, Any]: 

190 """Run one keep-alive pass on the active jar and return its health. 

191 

192 Never raises: a network failure becomes ``state='error'``. 

193 """ 

194 

195 with _run_lock: 

196 return _refresh_locked(store) 

197 

198 

199def _refresh_locked(store: CookiesStore) -> dict[str, Any]: 

200 path = store.active_path() 

201 if path is None or not path.is_file(): 

202 store.health = dict(UNKNOWN_HEALTH) 

203 return dict(store.health) 

204 

205 before = load_jar(path) 

206 if not _has_account_cookie(before, 'youtube.com') and not ( 

207 _has_account_cookie(before, 'google.com') 

208 ): 

209 _set_health( 

210 store, 

211 state='no_account', 

212 checked_at=_now(), 

213 message='No signed-in Google session in this cookies.txt', 

214 ) 

215 return dict(store.health) 

216 

217 session = _new_session() 

218 session.headers.update({ 

219 'User-Agent': _USER_AGENT, 

220 'Accept-Language': 'en-US,en;q=0.9', 

221 }) 

222 for cookie in before.values(): 

223 session.cookies.set_cookie(cookie) 

224 

225 if _has_account_cookie(before, 'google.com'): 

226 _rotate(session, 'accounts.google.com') 

227 if _has_account_cookie(before, 'youtube.com'): 

228 _rotate(session, 'accounts.youtube.com') 

229 

230 try: 

231 response = session.get('https://www.youtube.com/', timeout=_TIMEOUT) 

232 page = response.text 

233 except requests.RequestException as exc: 

234 _set_health( 

235 store, 

236 state='error', 

237 checked_at=_now(), 

238 message=f'YouTube could not be reached: {type(exc).__name__}', 

239 ) 

240 return dict(store.health) 

241 

242 after = {(c.domain, c.path, c.name): c for c in session.cookies} 

243 saved = merge_into(path, changed_cookies(before, after)) 

244 changes: dict[str, Any] = {'checked_at': _now()} 

245 if saved: 

246 changes['refreshed_at'] = changes['checked_at'] 

247 

248 match = _LOGGED_IN_RE.search(page) 

249 if match is None: 

250 changes |= { 

251 'state': 'error', 

252 'message': ( 

253 f'YouTube answered HTTP {response.status_code} without ' 

254 'its usual page' 

255 ), 

256 } 

257 elif match.group(1) == 'true': 

258 changes |= {'state': 'valid', 'message': None} 

259 else: 

260 changes |= { 

261 'state': 'signed_out', 

262 'message': 'YouTube signed this session out', 

263 } 

264 _set_health(store, **changes) 

265 return dict(store.health) 

266 

267 

268async def cookie_refresh_loop(store: CookiesStore) -> None: 

269 """Background task: :func:`refresh_cookies` every few minutes.""" 

270 

271 minutes = interval_minutes() 

272 if minutes == 0: 

273 logger.info('YouTube cookie keep-alive disabled ({}=0)', ENV_INTERVAL) 

274 return 

275 await asyncio.sleep(_STARTUP_DELAY_SECONDS) 

276 while True: 

277 try: 

278 if store.active_path() is not None: 

279 await asyncio.to_thread(refresh_cookies, store) 

280 except asyncio.CancelledError: 

281 raise 

282 except Exception: 

283 logger.opt(exception=True).warning('Cookie keep-alive failed') 

284 await asyncio.sleep(minutes * 60)