OpenCloud
/
PeerTube
mirror of https://github.com/Chocobozzz/PeerTube
1
0
Fork 0
Code Issues Releases Wiki Activity

Update P2P media loader peer version

pull/1765/head
Chocobozzz 2019-04-08 11:13:49 +02:00
parent 14aed608f5
commit ae9bbed46d
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
9 changed files with 121 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server.ts
View File

@ -103,6 +103,7 @@ import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-upd
import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
import { PeerTubeSocket } from './server/lib/peertube-socket'
import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
// ----------- Command line -----------
@ -233,6 +234,9 @@ async function startApplication () {
PeerTubeSocket.Instance.init(server)
updateStreamingPlaylistsInfohashesIfNeeded()
.catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
// Make server listening
server.listen(port, hostname, () => {
logger.info('Server listening on %s:%d', hostname, port)

2
server/helpers/core-utils.ts
View File

@ -58,7 +58,7 @@ export function parseDuration (duration: number | string): number {
}
}
throw new Error('Duration could not be properly parsed')
throw new Error(`Duration ${duration} could not be properly parsed`)
}
export function parseBytes (value: string | number): number {

3
server/helpers/video.ts
View File

@ -1,7 +1,4 @@
import { CONFIG } from '../initializers'
import { VideoModel } from '../models/video/video'
import { UserRight } from '../../shared'
import { UserModel } from '../models/account/user'
type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none'

5
server/initializers/constants.ts
View File

@ -18,7 +18,7 @@ let config: IConfig = require('config')
// ---------------------------------------------------------------------------
const LAST_MIGRATION_VERSION = 350
const LAST_MIGRATION_VERSION = 355
// ---------------------------------------------------------------------------
@ -726,6 +726,8 @@ const TRACKER_RATE_LIMITS = {
ANNOUNCES_PER_IP: 30 // maximum announces for all our torrents in the interval
}
const P2P_MEDIA_LOADER_PEER_VERSION = 2
// ---------------------------------------------------------------------------
// Special constants for a test instance
@ -772,6 +774,7 @@ updateWebserverUrls()
export {
API_VERSION,
HLS_REDUNDANCY_DIRECTORY,
P2P_MEDIA_LOADER_PEER_VERSION,
AVATARS_SIZE,
ACCEPT_HEADERS,
BCRYPT_SALT_SIZE,

41
server/initializers/migrations/0355-p2p-peer-version.ts Normal file
View File

@ -0,0 +1,41 @@
import * as Sequelize from 'sequelize'
async function up (utils: {
transaction: Sequelize.Transaction,
queryInterface: Sequelize.QueryInterface,
sequelize: Sequelize.Sequelize,
db: any
}): Promise<void> {
{
const data = {
type: Sequelize.INTEGER,
allowNull: true,
defaultValue: null
}
await utils.queryInterface.addColumn('videoStreamingPlaylist', 'p2pMediaLoaderPeerVersion', data)
}
{
const query = `UPDATE "videoStreamingPlaylist" SET "p2pMediaLoaderPeerVersion" = 0;`
await utils.sequelize.query(query)
}
{
const data = {
type: Sequelize.INTEGER,
allowNull: false,
defaultValue: null
}
await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'p2pMediaLoaderPeerVersion', data)
}
}
function down (options) {
throw new Error('Not implemented.')
}
export {
up,
down
}

22
server/lib/activitypub/videos.ts
View File

@ -290,7 +290,11 @@ async function updateVideoFromAP (options: {
}
{
const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject)
const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(
options.video,
options.videoObject,
options.video.VideoFiles
)
const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
// Remove video files that do not exist anymore
@ -449,9 +453,9 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
}
const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
await Promise.all(videoFilePromises)
const videoFiles = await Promise.all(videoFilePromises)
const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject)
const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
await Promise.all(playlistPromises)
@ -575,20 +579,12 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
return attributes
}
function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) {
const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
if (playlistUrls.length === 0) return []
const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
for (const playlistUrlObject of playlistUrls) {
const p2pMediaLoaderInfohashes = playlistUrlObject.tag
.filter(t => t.type === 'Infohash')
.map(t => t.name)
if (p2pMediaLoaderInfohashes.length === 0) {
logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
continue
}
const segmentsSha256UrlObject = playlistUrlObject.tag
.find(t => {
return isAPPlaylistSegmentHashesUrlObject(t)
@ -602,7 +598,7 @@ function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObj
type: VideoStreamingPlaylistType.HLS,
playlistUrl: playlistUrlObject.href,
segmentsSha256Url: segmentsSha256UrlObject.href,
p2pMediaLoaderInfohashes,
p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
videoId: video.id
}

22
server/lib/hls.ts
View File

@ -1,6 +1,6 @@
import { VideoModel } from '../models/video/video'
import { basename, join, dirname } from 'path'
import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY } from '../initializers'
import { basename, dirname, join } from 'path'
import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, sequelizeTypescript } from '../initializers'
import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
import { getVideoFileSize } from '../helpers/ffmpeg-utils'
import { sha256 } from '../helpers/core-utils'
@ -9,6 +9,21 @@ import { logger } from '../helpers/logger'
import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
import { generateRandomString } from '../helpers/utils'
import { flatten, uniq } from 'lodash'
import { VideoFileModel } from '../models/video/video-file'
async function updateStreamingPlaylistsInfohashesIfNeeded () {
const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
// Use separate SQL queries, because we could have many videos to update
for (const playlist of playlistsToUpdate) {
await sequelizeTypescript.transaction(async t => {
const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
playlist.p2pMediaLoaderInfohashes = await VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
await playlist.save({ transaction: t })
})
}
}
async function updateMasterHLSPlaylist (video: VideoModel) {
const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
@ -159,7 +174,8 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
export {
updateMasterHLSPlaylist,
updateSha256Segments,
downloadPlaylistSegments
downloadPlaylistSegments,
updateStreamingPlaylistsInfohashesIfNeeded
}
// ---------------------------------------------------------------------------

24
server/models/video/video-file.ts
View File

@ -23,6 +23,7 @@ import { throwIfNotValid } from '../utils'
import { VideoModel } from './video'
import * as Sequelize from 'sequelize'
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
@Table({
tableName: 'videoFile',
@ -120,6 +121,29 @@ export class VideoFileModel extends Model<VideoFileModel> {
return VideoFileModel.findByPk(id, options)
}
static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Sequelize.Transaction) {
const query = {
include: [
{
model: VideoModel.unscoped(),
required: true,
include: [
{
model: VideoStreamingPlaylistModel.unscoped(),
required: true,
where: {
id: streamingPlaylistId
}
}
]
}
],
transaction
}
return VideoFileModel.findAll(query)
}
static async getStats () {
let totalLocalVideoFilesSize = await VideoFileModel.sum('size', {
include: [

22
server/models/video/video-streaming-playlist.ts
View File

@ -6,7 +6,7 @@ import * as Sequelize from 'sequelize'
import { VideoRedundancyModel } from '../redundancy/video-redundancy'
import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers'
import { CONSTRAINTS_FIELDS, STATIC_PATHS, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers'
import { VideoFileModel } from './video-file'
import { join } from 'path'
import { sha1 } from '../../helpers/core-utils'
@ -49,6 +49,10 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
@Column(DataType.ARRAY(DataType.STRING))
p2pMediaLoaderInfohashes: string[]
@AllowNull(false)
@Column
p2pMediaLoaderPeerVersion: number
@AllowNull(false)
@Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
@Column
@ -92,14 +96,26 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) {
const hashes: string[] = []
// https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97
// https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L115
for (let i = 0; i < videoFiles.length; i++) {
hashes.push(sha1(`1${playlistUrl}+V${i}`))
hashes.push(sha1(`${P2P_MEDIA_LOADER_PEER_VERSION}${playlistUrl}+V${i}`))
}
return hashes
}
static listByIncorrectPeerVersion () {
const query = {
where: {
p2pMediaLoaderPeerVersion: {
[Sequelize.Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION
}
}
}
return VideoStreamingPlaylistModel.findAll(query)
}
static loadWithVideo (id: number) {
const options = {
include: [