From d78d3091ed93c4243c98c856fd5eb534be0e624c Mon Sep 17 00:00:00 2001 From: LaptopsPlural <88594453+LaptopsPlural@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:41:01 +0000 Subject: [PATCH] Use InnerTube instead of RSS for channel video refresh Replace /feeds/videos.xml parsing in fetch_channel with the existing get_about_info and Channel::Tabs.get_videos helpers. Preserve stored published/updated timestamps on conflict so relative InnerTube dates do not drift and Atom feeds do not churn on unchanged videos. Mark channels deleted via NotFoundException from InnerTube about errors. Fixes #2410 --- src/invidious/channels/about.cr | 7 +- src/invidious/channels/channels.cr | 137 +++++++-------------- src/invidious/database/channels.cr | 17 ++- src/invidious/jobs/refresh_channels_job.cr | 2 +- 4 files changed, 67 insertions(+), 96 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 247f628a0..da7ec452c 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -19,6 +19,11 @@ record AboutChannel, verified : Bool, is_age_gated : Bool +private DELETED_CHANNEL_ERROR_MESSAGES = { + "This channel does not exist.", + "This account has been terminated for a violation of YouTube's Terms of Service.", +} + def get_about_info(ucid) : AboutChannel begin # Fetch channel information from channel home page @@ -29,7 +34,7 @@ def get_about_info(ucid) : AboutChannel if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR" error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s - if error_message == "This channel does not exist." + if DELETED_CHANNEL_ERROR_MESSAGES.includes?(error_message) raise NotFoundException.new(error_message) else raise InfoException.new(error_message) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 64f0484df..cc6614354 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -159,127 +159,65 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.debug("fetch_channel: #{ucid}") LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") - namespaces = { - "yt" => "http://www.youtube.com/xml/schemas/2015", - "media" => "http://search.yahoo.com/mrss/", - "default" => "http://www.w3.org/2005/Atom", - } + # Channel RSS (/feeds/videos.xml) is unreliable and rate-limited. Use the + # existing InnerTube-backed helpers instead (see #2410). + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel information") + about = get_about_info(ucid) - LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") - rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body - LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") - rss = XML.parse(rss) - - author = rss.xpath_node("//default:feed/default:title", namespaces) - if !author - raise InfoException.new("Deleted or invalid channel") - end - - author = author.content - - # Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - if author.ends_with?(" - Topic") || - {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author - auto_generated = true - end - - LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") + LOGGER.trace("fetch_channel: #{ucid} : author = #{about.author}, auto_generated = #{about.auto_generated}") channel = InvidiousChannel.new({ - id: ucid, - author: author, + id: about.ucid, + author: about.author, updated: Time.utc, deleted: false, subscribed: nil, }) LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") - videos, continuation = IV::Channel::Tabs.get_videos(channel) + videos, continuation = IV::Channel::Tabs.get_videos(about) - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") - rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| - video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content - title = entry.xpath_node("default:title", namespaces).not_nil!.content + LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel tabs") + videos.select(SearchVideo).each do |search_video| + video = channel_video_from_search(search_video) - published = Time.parse_rfc3339( - entry.xpath_node("default:published", namespaces).not_nil!.content + LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Updating or inserting video") + + # InnerTube only exposes relative publication labels. Keep previously stored + # published/updated values on conflict so refreshes do not drift timestamps + # or churn Atom for unchanged entries. + was_insert = Invidious::Database::ChannelVideos.insert( + video, + update_published: false, + update_updated: false, ) - updated = Time.parse_rfc3339( - entry.xpath_node("default:updated", namespaces).not_nil!.content - ) - - author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content - ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content - - views = entry - .xpath_node("media:group/media:community/media:statistics", namespaces) - .try &.["views"]?.try &.to_i64? || 0_i64 - - channel_video = videos - .select(SearchVideo) - .select(&.id.== video_id)[0]? - - length_seconds = channel_video.try &.length_seconds - length_seconds ||= 0 - - live_now = channel_video.try &.badges.live_now? - live_now ||= false - - premiere_timestamp = channel_video.try &.premiere_timestamp - - video = ChannelVideo.new({ - id: video_id, - title: title, - published: published, - updated: updated, - ucid: ucid, - author: author, - length_seconds: length_seconds, - live_now: live_now, - premiere_timestamp: premiere_timestamp, - views: views, - }) - - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") - - # We don't include the 'premiere_timestamp' here because channel pages don't include them, - # meaning the above timestamp is always null - was_insert = Invidious::Database::ChannelVideos.insert(video) if was_insert - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") + LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Inserted, updating subscriptions") NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) else - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") + LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Updated") end end if pull_all_videos loop do # Keep fetching videos using the continuation token retrieved earlier - videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) + videos, continuation = IV::Channel::Tabs.get_videos(about, continuation: continuation) count = 0 - videos.select(SearchVideo).each do |video| + videos.select(SearchVideo).each do |search_video| count += 1 - video = ChannelVideo.new({ - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.badges.live_now?, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) + video = channel_video_from_search(search_video) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. if Time.utc - video.published > 1.minute - was_insert = Invidious::Database::ChannelVideos.insert(video) + was_insert = Invidious::Database::ChannelVideos.insert( + video, + update_published: false, + update_updated: false, + ) if was_insert NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) end @@ -294,3 +232,18 @@ def fetch_channel(ucid, pull_all_videos : Bool) channel.updated = Time.utc return channel end + +private def channel_video_from_search(video : SearchVideo) : ChannelVideo + ChannelVideo.new({ + id: video.id, + title: video.title, + published: video.published, + updated: Time.utc, + ucid: video.ucid, + author: video.author, + length_seconds: video.length_seconds, + live_now: video.badges.live_now?, + premiere_timestamp: video.premiere_timestamp, + views: video.views, + }) +end diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr index df44e485d..fc2afd53a 100644 --- a/src/invidious/database/channels.cr +++ b/src/invidious/database/channels.cr @@ -98,18 +98,31 @@ module Invidious::Database::ChannelVideos # ------------------- # This function returns the status of the query (i.e: success?) - def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool + # update_published / update_updated control whether those columns are + # overwritten on conflict. InnerTube refreshes pass false so relative + # publication labels and refresh-time "updated" values do not drift or + # churn Atom feeds for unchanged videos. + def insert( + video : ChannelVideo, + with_premiere_timestamp : Bool = false, + *, + update_published : Bool = true, + update_updated : Bool = true, + ) : Bool if with_premiere_timestamp last_items = "premiere_timestamp = $9, views = $10" else last_items = "views = $10" end + published_item = update_published ? "published = $3," : "" + updated_item = update_updated ? "updated = $4," : "" + request = <<-SQL INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (id) DO UPDATE - SET title = $2, published = $3, updated = $4, ucid = $5, + SET title = $2, #{published_item} #{updated_item} ucid = $5, author = $6, length_seconds = $7, live_now = $8, #{last_items} RETURNING (xmax=0) AS was_insert SQL diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 5c5ecc837..283bd951d 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -43,7 +43,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end rescue ex LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}") - if ex.message == "Deleted or invalid channel" + if ex.is_a?(NotFoundException) Invidious::Database::Channels.update_mark_deleted(id) else lim_fibers = 1