1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
|
enum VideoType
Video
Livestream
Scheduled
end
struct Video
include DB::Serializable
# Version of the JSON structure
# It prevents us from loading an incompatible version from cache
# (either newer or older, if instances with different versions run
# concurrently, e.g during a version upgrade rollout).
#
# NOTE: don't forget to bump this number if any change is made to
# the `params` structure in videos/parser.cr!!!
#
SCHEMA_VERSION = 2
property id : String
@[DB::Field(converter: Video::JSONConverter)]
property info : Hash(String, JSON::Any)
property updated : Time
@[DB::Field(ignore: true)]
@captions = [] of Invidious::Videos::Captions::Metadata
@[DB::Field(ignore: true)]
property adaptive_fmts : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property fmt_stream : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property description : String?
module JSONConverter
def self.from_rs(rs)
JSON.parse(rs.read(String)).as_h
end
end
# Methods for API v1 JSON
def to_json(locale : String?, json : JSON::Builder)
Invidious::JSONify::APIv1.video(self, json, locale: locale)
end
# TODO: remove the locale and follow the crystal convention
def to_json(locale : String?, _json : Nil)
JSON.build do |json|
Invidious::JSONify::APIv1.video(self, json, locale: locale)
end
end
def to_json(json : JSON::Builder | Nil = nil)
to_json(nil, json)
end
# Misc methods
def video_type : VideoType
video_type = info["videoType"]?.try &.as_s || "video"
return VideoType.parse?(video_type) || VideoType::Video
end
def schema_version : Int
return info["version"]?.try &.as_i || 1
end
def published : Time
return info["published"]?
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
end
def published=(other : Time)
info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
end
def live_now
return (self.video_type == VideoType::Livestream)
end
def post_live_dvr
return info["isPostLiveDvr"].as_bool
end
def premiere_timestamp : Time?
info
.dig?("microformat", "playerMicroformatRenderer", "liveBroadcastDetails", "startTimestamp")
.try { |t| Time.parse_rfc3339(t.as_s) }
end
def related_videos
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
end
# Methods for parsing streaming data
def convert_url(fmt)
if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) }
sp = cfr["sp"]
url = URI.parse(cfr["url"])
params = url.query_params
LOGGER.debug("Videos: Decoding '#{cfr}'")
unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"])
params[sp] = unsig if unsig
else
url = URI.parse(fmt["url"].as_s)
params = url.query_params
end
n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"])
params["n"] = n if n
if token = CONFIG.po_token
params["pot"] = token
end
params["host"] = url.host.not_nil!
if region = self.info["region"]?.try &.as_s
params["region"] = region
end
url.query_params = params
LOGGER.trace("Videos: new url is '#{url}'")
return url.to_s
rescue ex
LOGGER.debug("Videos: Error when parsing video URL")
LOGGER.trace(ex.inspect_with_backtrace)
return ""
end
def fmt_stream
return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
fmt_stream = info.dig?("streamingData", "formats")
.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
fmt_stream.each do |fmt|
fmt["url"] = JSON::Any.new(self.convert_url(fmt))
end
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
@fmt_stream = fmt_stream
return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
def adaptive_fmts
return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
fmt_stream = info.dig("streamingData", "adaptiveFormats")
.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
fmt_stream.each do |fmt|
fmt["url"] = JSON::Any.new(self.convert_url(fmt))
end
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
@adaptive_fmts = fmt_stream
return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video")
end
def audio_streams
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
# Misc. methods
def storyboards
container = info.dig?("storyboards") || JSON::Any.new("{}")
return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds)
end
def paid
return (self.reason || "").includes? "requires payment"
end
def premium
keywords.includes? "YouTube Red"
end
def captions : Array(Invidious::Videos::Captions::Metadata)
if @captions.empty? && @info.has_key?("captions")
@captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"])
end
return @captions
end
def hls_manifest_url : String?
info.dig?("streamingData", "hlsManifestUrl").try &.as_s
end
def dash_manifest_url : String?
raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s
return nil if raw_dash_url.nil?
# Use manifest v5 parameter to reduce file size
# See https://github.com/iv-org/invidious/issues/4186
dash_url = URI.parse(raw_dash_url)
dash_query = dash_url.query || ""
if dash_query.empty?
dash_url.path = "#{dash_url.path}/mpd_version/5"
else
dash_url.query = "#{dash_query}&mpd_version=5"
end
return dash_url.to_s
end
def genre_url : String?
info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
end
def vr? : Bool?
return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
end
def projection_type : String?
return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
end
def reason : String?
info["reason"]?.try &.as_s
end
def music : Array(VideoMusic)
info["music"].as_a.map { |music_json|
VideoMusic.new(
music_json["song"].as_s,
music_json["album"].as_s,
music_json["artist"].as_s,
music_json["license"].as_s
)
}
end
# Macros defining getters/setters for various types of data
private macro getset_string(name)
# Return {{name.stringify}} from `info`
def {{name.id.underscore}} : String
return info[{{name.stringify}}]?.try &.as_s || ""
end
# Update {{name.stringify}} into `info`
def {{name.id.underscore}}=(value : String)
info[{{name.stringify}}] = JSON::Any.new(value)
end
{% if flag?(:debug_macros) %} {{debug}} {% end %}
end
private macro getset_string_array(name)
# Return {{name.stringify}} from `info`
def {{name.id.underscore}} : Array(String)
return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String
end
# Update {{name.stringify}} into `info`
def {{name.id.underscore}}=(value : Array(String))
info[{{name.stringify}}] = JSON::Any.new(value)
end
{% if flag?(:debug_macros) %} {{debug}} {% end %}
end
{% for op, type in {i32: Int32, i64: Int64} %}
private macro getset_{{op}}(name)
def \{{name.id.underscore}} : {{type}}
return info[\{{name.stringify}}]?.try &.as_i64.to_{{op}} || 0_{{op}}
end
def \{{name.id.underscore}}=(value : Int)
info[\{{name.stringify}}] = JSON::Any.new(value.to_i64)
end
\{% if flag?(:debug_macros) %} \{{debug}} \{% end %}
end
{% end %}
private macro getset_bool(name)
# Return {{name.stringify}} from `info`
def {{name.id.underscore}} : Bool
return info[{{name.stringify}}]?.try &.as_bool || false
end
# Update {{name.stringify}} into `info`
def {{name.id.underscore}}=(value : Bool)
info[{{name.stringify}}] = JSON::Any.new(value)
end
{% if flag?(:debug_macros) %} {{debug}} {% end %}
end
# Macro to generate ? and = accessor methods for attributes in `info`
private macro predicate_bool(method_name, name)
# Return {{name.stringify}} from `info`
def {{method_name.id.underscore}}? : Bool
return info[{{name.stringify}}]?.try &.as_bool || false
end
# Update {{name.stringify}} into `info`
def {{method_name.id.underscore}}=(value : Bool)
info[{{name.stringify}}] = JSON::Any.new(value)
end
{% if flag?(:debug_macros) %} {{debug}} {% end %}
end
# Method definitions, using the macros above
getset_string author
getset_string authorThumbnail
getset_string description
getset_string descriptionHtml
getset_string genre
getset_string genreUcid
getset_string license
getset_string shortDescription
getset_string subCountText
getset_string title
getset_string ucid
getset_string_array allowedRegions
getset_string_array keywords
getset_i32 lengthSeconds
getset_i64 likes
getset_i64 views
# TODO: Make predicate_bool the default as to adhere to Crystal conventions
getset_bool allowRatings
getset_bool authorVerified
getset_bool isFamilyFriendly
getset_bool isListed
predicate_bool upcoming, isUpcoming
end
def get_video(id, refresh = true, region = nil, force_refresh = false)
if (video = Invidious::Database::Videos.select(id)) && !region
# If record was last updated over 10 minutes ago, or video has since premiered,
# refresh (expire param in response lasts for 6 hours)
if (refresh &&
(Time.utc - video.updated > 10.minutes) ||
(video.premiere_timestamp.try &.< Time.utc)) ||
force_refresh ||
video.schema_version != Video::SCHEMA_VERSION # cache control
begin
video = fetch_video(id, region)
Invidious::Database::Videos.update(video)
rescue ex
Invidious::Database::Videos.delete(id)
raise ex
end
end
else
video = fetch_video(id, region)
Invidious::Database::Videos.insert(video) if !region
end
return video
rescue DB::Error
# Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
# Note: All DB errors inherit from `DB::Error`
return fetch_video(id, region)
end
def fetch_video(id, region)
info = extract_video_info(video_id: id)
if reason = info["reason"]?
if reason == "Video unavailable"
raise NotFoundException.new(reason.as_s || "")
elsif !reason.as_s.starts_with? "Premieres"
# dont error when it's a premiere.
# we already parsed most of the data and display the premiere date
raise InfoException.new(reason.as_s || "")
end
end
video = Video.new({
id: id,
info: info,
updated: Time.utc,
})
return video
end
def process_continuation(query, plid, id)
continuation = nil
if plid
if index = query["index"]?.try &.to_i?
continuation = index
else
continuation = id
end
continuation ||= 0
end
continuation
end
def build_thumbnails(id)
return {
{host: HOST_URL, height: 720, width: 1280, name: "maxres", url: "maxres"},
{host: HOST_URL, height: 720, width: 1280, name: "maxresdefault", url: "maxresdefault"},
{host: HOST_URL, height: 480, width: 640, name: "sddefault", url: "sddefault"},
{host: HOST_URL, height: 360, width: 480, name: "high", url: "hqdefault"},
{host: HOST_URL, height: 180, width: 320, name: "medium", url: "mqdefault"},
{host: HOST_URL, height: 90, width: 120, name: "default", url: "default"},
{host: HOST_URL, height: 90, width: 120, name: "start", url: "1"},
{host: HOST_URL, height: 90, width: 120, name: "middle", url: "2"},
{host: HOST_URL, height: 90, width: 120, name: "end", url: "3"},
}
end
|