diff options
-rw-r--r-- | src/invidious.cr | 66 | ||||
-rw-r--r-- | src/invidious/helpers/handlers.cr | 29 | ||||
-rw-r--r-- | src/invidious/helpers/helpers.cr | 4 | ||||
-rw-r--r-- | src/invidious/helpers/jobs.cr | 82 | ||||
-rw-r--r-- | src/invidious/helpers/proxy.cr | 8 | ||||
-rw-r--r-- | src/invidious/helpers/utils.cr | 5 | ||||
-rw-r--r-- | src/invidious/mixes.cr | 2 | ||||
-rw-r--r-- | src/invidious/trending.cr | 2 |
8 files changed, 170 insertions, 28 deletions
diff --git a/src/invidious.cr b/src/invidious.cr index 6a197795..b0a99d21 100644 --- a/src/invidious.cr +++ b/src/invidious.cr | |||
@@ -5568,7 +5568,7 @@ get "/videoplayback" do |env| | |||
5568 | next env.redirect location | 5568 | next env.redirect location |
5569 | end | 5569 | end |
5570 | 5570 | ||
5571 | IO.copy(response.body_io, env.response) | 5571 | IO.copy response.body_io, env.response |
5572 | end | 5572 | end |
5573 | rescue ex | 5573 | rescue ex |
5574 | end | 5574 | end |
@@ -5865,6 +5865,69 @@ get "/Captcha" do |env| | |||
5865 | response.body | 5865 | response.body |
5866 | end | 5866 | end |
5867 | 5867 | ||
5868 | connect "*" do |env| | ||
5869 | if CONFIG.proxy_address.empty? | ||
5870 | env.response.status_code = 400 | ||
5871 | next | ||
5872 | end | ||
5873 | |||
5874 | url = env.request.headers["Host"]?.try { |u| u.split(":") } | ||
5875 | host = url.try &.[0]? | ||
5876 | port = url.try &.[1]? | ||
5877 | |||
5878 | host = "www.google.com" if !host || host.empty? | ||
5879 | port = "443" if !port || port.empty? | ||
5880 | |||
5881 | # if env.request.internal_uri | ||
5882 | # env.request.internal_uri.not_nil!.path = "#{host}:#{port}" | ||
5883 | # end | ||
5884 | |||
5885 | user, pass = env.request.headers["Proxy-Authorization"]? | ||
5886 | .try { |i| i.lchop("Basic ") } | ||
5887 | .try { |i| Base64.decode_string(i) } | ||
5888 | .try &.split(":", 2) || {nil, nil} | ||
5889 | |||
5890 | if CONFIG.proxy_user != user || CONFIG.proxy_pass != pass | ||
5891 | env.response.status_code = 403 | ||
5892 | next | ||
5893 | end | ||
5894 | |||
5895 | begin | ||
5896 | upstream = TCPSocket.new(host, port) | ||
5897 | rescue ex | ||
5898 | logger.puts("Exception: #{ex.message}") | ||
5899 | env.response.status_code = 400 | ||
5900 | next | ||
5901 | end | ||
5902 | |||
5903 | env.response.reset | ||
5904 | env.response.upgrade do |downstream| | ||
5905 | downstream = downstream.as(TCPSocket) | ||
5906 | downstream.sync = true | ||
5907 | |||
5908 | spawn do | ||
5909 | begin | ||
5910 | bytes = 1 | ||
5911 | while bytes != 0 | ||
5912 | bytes = IO.copy upstream, downstream | ||
5913 | end | ||
5914 | rescue ex | ||
5915 | end | ||
5916 | end | ||
5917 | |||
5918 | begin | ||
5919 | bytes = 1 | ||
5920 | while bytes != 0 | ||
5921 | bytes = IO.copy downstream, upstream | ||
5922 | end | ||
5923 | rescue ex | ||
5924 | ensure | ||
5925 | upstream.close | ||
5926 | downstream.close | ||
5927 | end | ||
5928 | end | ||
5929 | end | ||
5930 | |||
5868 | # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos | 5931 | # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos |
5869 | get "/watch_videos" do |env| | 5932 | get "/watch_videos" do |env| |
5870 | response = YT_POOL.client &.get(env.request.resource) | 5933 | response = YT_POOL.client &.get(env.request.resource) |
@@ -5939,6 +6002,7 @@ end | |||
5939 | public_folder "assets" | 6002 | public_folder "assets" |
5940 | 6003 | ||
5941 | Kemal.config.powered_by_header = false | 6004 | Kemal.config.powered_by_header = false |
6005 | add_handler ProxyHandler.new | ||
5942 | add_handler FilteredCompressHandler.new | 6006 | add_handler FilteredCompressHandler.new |
5943 | add_handler APIHandler.new | 6007 | add_handler APIHandler.new |
5944 | add_handler AuthHandler.new | 6008 | add_handler AuthHandler.new |
diff --git a/src/invidious/helpers/handlers.cr b/src/invidious/helpers/handlers.cr index 87b10bc9..f3b99b7d 100644 --- a/src/invidious/helpers/handlers.cr +++ b/src/invidious/helpers/handlers.cr | |||
@@ -212,3 +212,32 @@ class DenyFrame < Kemal::Handler | |||
212 | call_next env | 212 | call_next env |
213 | end | 213 | end |
214 | end | 214 | end |
215 | |||
216 | class ProxyHandler < Kemal::Handler | ||
217 | def call(env) | ||
218 | if env.request.headers["Proxy-Authorization"]? && env.request.method != "CONNECT" | ||
219 | user, pass = env.request.headers["Proxy-Authorization"]? | ||
220 | .try { |i| i.lchop("Basic ") } | ||
221 | .try { |i| Base64.decode_string(i) } | ||
222 | .try &.split(":", 2) || {nil, nil} | ||
223 | |||
224 | if CONFIG.proxy_user != user || CONFIG.proxy_pass != pass | ||
225 | env.response.status_code = 403 | ||
226 | return | ||
227 | end | ||
228 | |||
229 | HTTP::Client.exec(env.request.method, "#{env.request.headers["Host"]?}#{env.request.resource}", env.request.headers, env.request.body) do |response| | ||
230 | response.headers.each do |key, value| | ||
231 | if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "transfer-encoding" | ||
232 | env.response.headers[key] = value | ||
233 | end | ||
234 | end | ||
235 | IO.copy response.body_io, env.response | ||
236 | end | ||
237 | env.response.close | ||
238 | return | ||
239 | else | ||
240 | call_next env | ||
241 | end | ||
242 | end | ||
243 | end | ||
diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 96d14737..ba095bc3 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr | |||
@@ -263,6 +263,10 @@ struct Config | |||
263 | admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports | 263 | admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports |
264 | cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format | 264 | cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format |
265 | captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha | 265 | captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha |
266 | proxy_address: {type: String, default: ""}, | ||
267 | proxy_port: {type: Int32, default: 8080}, | ||
268 | proxy_user: {type: String, default: ""}, | ||
269 | proxy_pass: {type: String, default: ""}, | ||
266 | }) | 270 | }) |
267 | end | 271 | end |
268 | 272 | ||
diff --git a/src/invidious/helpers/jobs.cr b/src/invidious/helpers/jobs.cr index c6e0ef42..02c3ab05 100644 --- a/src/invidious/helpers/jobs.cr +++ b/src/invidious/helpers/jobs.cr | |||
@@ -249,15 +249,34 @@ def bypass_captcha(captcha_key, logger) | |||
249 | end | 249 | end |
250 | 250 | ||
251 | headers = response.cookies.add_request_headers(HTTP::Headers.new) | 251 | headers = response.cookies.add_request_headers(HTTP::Headers.new) |
252 | 252 | captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com")) | |
253 | response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: { | 253 | captcha_client.family = CONFIG.force_resolve || Socket::Family::INET |
254 | "clientKey" => CONFIG.captcha_key, | 254 | if !CONFIG.proxy_address.empty? |
255 | "task" => { | 255 | response = JSON.parse(captcha_client.post("/createTask", body: { |
256 | "type" => "NoCaptchaTaskProxyless", | 256 | "clientKey" => CONFIG.captcha_key, |
257 | "websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999", | 257 | "task" => { |
258 | "websiteKey" => site_key, | 258 | "type" => "NoCaptchaTask", |
259 | }, | 259 | "websiteURL" => "https://www.youtube.com#{path}", |
260 | }.to_json).body) | 260 | "websiteKey" => site_key, |
261 | "proxyType" => "http", | ||
262 | "proxyAddress" => CONFIG.proxy_address, | ||
263 | "proxyPort" => CONFIG.proxy_port, | ||
264 | "proxyLogin" => CONFIG.proxy_user, | ||
265 | "proxyPassword" => CONFIG.proxy_pass, | ||
266 | "userAgent" => headers["user-agent"], | ||
267 | }, | ||
268 | }.to_json).body) | ||
269 | else | ||
270 | response = JSON.parse(captcha_client.post("/createTask", body: { | ||
271 | "clientKey" => CONFIG.captcha_key, | ||
272 | "task" => { | ||
273 | "type" => "NoCaptchaTaskProxyless", | ||
274 | "websiteURL" => "https://www.youtube.com#{path}", | ||
275 | "websiteKey" => site_key, | ||
276 | "userAgent" => headers["user-agent"], | ||
277 | }, | ||
278 | }.to_json).body) | ||
279 | end | ||
261 | 280 | ||
262 | raise response["error"].as_s if response["error"]? | 281 | raise response["error"].as_s if response["error"]? |
263 | task_id = response["taskId"].as_i | 282 | task_id = response["taskId"].as_i |
@@ -265,7 +284,7 @@ def bypass_captcha(captcha_key, logger) | |||
265 | loop do | 284 | loop do |
266 | sleep 10.seconds | 285 | sleep 10.seconds |
267 | 286 | ||
268 | response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: { | 287 | response = JSON.parse(captcha_client.post("/getTaskResult", body: { |
269 | "clientKey" => CONFIG.captcha_key, | 288 | "clientKey" => CONFIG.captcha_key, |
270 | "taskId" => task_id, | 289 | "taskId" => task_id, |
271 | }.to_json).body) | 290 | }.to_json).body) |
@@ -283,7 +302,11 @@ def bypass_captcha(captcha_key, logger) | |||
283 | yield response.cookies.select { |cookie| cookie.name != "PREF" } | 302 | yield response.cookies.select { |cookie| cookie.name != "PREF" } |
284 | elsif response.headers["Location"]?.try &.includes?("/sorry/index") | 303 | elsif response.headers["Location"]?.try &.includes?("/sorry/index") |
285 | location = response.headers["Location"].try { |u| URI.parse(u) } | 304 | location = response.headers["Location"].try { |u| URI.parse(u) } |
286 | headers = HTTP::Headers{":authority" => location.host.not_nil!} | 305 | headers = HTTP::Headers{ |
306 | ":authority" => location.host.not_nil!, | ||
307 | "origin" => "https://www.google.com", | ||
308 | "user-agent" => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36", | ||
309 | } | ||
287 | response = YT_POOL.client &.get(location.full_path, headers) | 310 | response = YT_POOL.client &.get(location.full_path, headers) |
288 | 311 | ||
289 | html = XML.parse_html(response.body) | 312 | html = XML.parse_html(response.body) |
@@ -297,14 +320,32 @@ def bypass_captcha(captcha_key, logger) | |||
297 | 320 | ||
298 | captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com")) | 321 | captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com")) |
299 | captcha_client.family = CONFIG.force_resolve || Socket::Family::INET | 322 | captcha_client.family = CONFIG.force_resolve || Socket::Family::INET |
300 | response = JSON.parse(captcha_client.post("/createTask", body: { | 323 | if !CONFIG.proxy_address.empty? |
301 | "clientKey" => CONFIG.captcha_key, | 324 | response = JSON.parse(captcha_client.post("/createTask", body: { |
302 | "task" => { | 325 | "clientKey" => CONFIG.captcha_key, |
303 | "type" => "NoCaptchaTaskProxyless", | 326 | "task" => { |
304 | "websiteURL" => location.to_s, | 327 | "type" => "NoCaptchaTask", |
305 | "websiteKey" => site_key, | 328 | "websiteURL" => location.to_s, |
306 | }, | 329 | "websiteKey" => site_key, |
307 | }.to_json).body) | 330 | "proxyType" => "http", |
331 | "proxyAddress" => CONFIG.proxy_address, | ||
332 | "proxyPort" => CONFIG.proxy_port, | ||
333 | "proxyLogin" => CONFIG.proxy_user, | ||
334 | "proxyPassword" => CONFIG.proxy_pass, | ||
335 | "userAgent" => headers["user-agent"], | ||
336 | }, | ||
337 | }.to_json).body) | ||
338 | else | ||
339 | response = JSON.parse(captcha_client.post("/createTask", body: { | ||
340 | "clientKey" => CONFIG.captcha_key, | ||
341 | "task" => { | ||
342 | "type" => "NoCaptchaTaskProxyless", | ||
343 | "websiteURL" => location.to_s, | ||
344 | "websiteKey" => site_key, | ||
345 | "userAgent" => headers["user-agent"], | ||
346 | }, | ||
347 | }.to_json).body) | ||
348 | end | ||
308 | 349 | ||
309 | raise response["error"].as_s if response["error"]? | 350 | raise response["error"].as_s if response["error"]? |
310 | task_id = response["taskId"].as_i | 351 | task_id = response["taskId"].as_i |
@@ -326,8 +367,7 @@ def bypass_captcha(captcha_key, logger) | |||
326 | 367 | ||
327 | inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s | 368 | inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s |
328 | headers["content-type"] = "application/x-www-form-urlencoded" | 369 | headers["content-type"] = "application/x-www-form-urlencoded" |
329 | headers["origin"] = "https://www.google.com" | 370 | headers["referer"] = location.to_s |
330 | headers["user-agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" | ||
331 | 371 | ||
332 | response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) | 372 | response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs) |
333 | headers = HTTP::Headers{ | 373 | headers = HTTP::Headers{ |
diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr index 4f415ba0..af114e29 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/helpers/proxy.cr | |||
@@ -1,3 +1,7 @@ | |||
1 | def connect(path : String, &block : HTTP::Server::Context -> _) | ||
2 | Kemal::RouteHandler::INSTANCE.add_route("CONNECT", path, &block) | ||
3 | end | ||
4 | |||
1 | # See https://github.com/crystal-lang/crystal/issues/2963 | 5 | # See https://github.com/crystal-lang/crystal/issues/2963 |
2 | class HTTPProxy | 6 | class HTTPProxy |
3 | getter proxy_host : String | 7 | getter proxy_host : String |
@@ -124,7 +128,7 @@ def get_nova_proxies(country_code = "US") | |||
124 | client.connect_timeout = 10.seconds | 128 | client.connect_timeout = 10.seconds |
125 | 129 | ||
126 | headers = HTTP::Headers.new | 130 | headers = HTTP::Headers.new |
127 | headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" | 131 | headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" |
128 | headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" | 132 | headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" |
129 | headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" | 133 | headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" |
130 | headers["Host"] = "www.proxynova.com" | 134 | headers["Host"] = "www.proxynova.com" |
@@ -161,7 +165,7 @@ def get_spys_proxies(country_code = "US") | |||
161 | client.connect_timeout = 10.seconds | 165 | client.connect_timeout = 10.seconds |
162 | 166 | ||
163 | headers = HTTP::Headers.new | 167 | headers = HTTP::Headers.new |
164 | headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" | 168 | headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" |
165 | headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" | 169 | headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" |
166 | headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" | 170 | headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9" |
167 | headers["Host"] = "spys.one" | 171 | headers["Host"] = "spys.one" |
diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 1fff206d..52ca0f82 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr | |||
@@ -2,11 +2,12 @@ require "lsquic" | |||
2 | require "pool/connection" | 2 | require "pool/connection" |
3 | 3 | ||
4 | def add_yt_headers(request) | 4 | def add_yt_headers(request) |
5 | request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" | 5 | return if request.resource.starts_with? "/sorry/index" |
6 | |||
7 | request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" | ||
6 | request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" | 8 | request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7" |
7 | request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" | 9 | request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" |
8 | request.headers["accept-language"] ||= "en-us,en;q=0.5" | 10 | request.headers["accept-language"] ||= "en-us,en;q=0.5" |
9 | return if request.resource.starts_with? "/sorry/index" | ||
10 | request.headers["x-youtube-client-name"] ||= "1" | 11 | request.headers["x-youtube-client-name"] ||= "1" |
11 | request.headers["x-youtube-client-version"] ||= "1.20180719" | 12 | request.headers["x-youtube-client-version"] ||= "1.20180719" |
12 | if !CONFIG.cookies.empty? | 13 | if !CONFIG.cookies.empty? |
diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 04a37b87..f5ff40ef 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr | |||
@@ -20,7 +20,7 @@ end | |||
20 | 20 | ||
21 | def fetch_mix(rdid, video_id, cookies = nil, locale = nil) | 21 | def fetch_mix(rdid, video_id, cookies = nil, locale = nil) |
22 | headers = HTTP::Headers.new | 22 | headers = HTTP::Headers.new |
23 | headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36" | 23 | headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" |
24 | 24 | ||
25 | if cookies | 25 | if cookies |
26 | headers = cookies.add_request_headers(headers) | 26 | headers = cookies.add_request_headers(headers) |
diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 017c42f5..4f80be64 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr | |||
@@ -1,6 +1,6 @@ | |||
1 | def fetch_trending(trending_type, region, locale) | 1 | def fetch_trending(trending_type, region, locale) |
2 | headers = HTTP::Headers.new | 2 | headers = HTTP::Headers.new |
3 | headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36" | 3 | headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" |
4 | 4 | ||
5 | region ||= "US" | 5 | region ||= "US" |
6 | region = region.upcase | 6 | region = region.upcase |