PeerTube/server/lib/friends.ts

429 lines
12 KiB
TypeScript
Raw Normal View History

2017-06-05 21:53:49 +02:00
import * as request from 'request'
2017-06-10 22:15:25 +02:00
import * as Sequelize from 'sequelize'
import * as Promise from 'bluebird'
2017-07-12 11:56:02 +02:00
import { join } from 'path'
2017-05-22 20:58:25 +02:00
import { database as db } from '../initializers/database'
2017-05-15 22:22:03 +02:00
import {
API_VERSION,
CONFIG,
REQUESTS_IN_PARALLEL,
REQUEST_ENDPOINTS,
REQUEST_ENDPOINT_ACTIONS,
2017-07-12 11:56:02 +02:00
REMOTE_SCHEME,
STATIC_PATHS
2017-05-15 22:22:03 +02:00
} from '../initializers'
import {
logger,
getMyPublicCert,
makeSecureRequest,
makeRetryRequest
2017-05-15 22:22:03 +02:00
} from '../helpers'
import {
RequestScheduler,
2017-06-10 22:15:25 +02:00
RequestSchedulerOptions,
2017-05-15 22:22:03 +02:00
RequestVideoQaduScheduler,
2017-06-10 22:15:25 +02:00
RequestVideoQaduSchedulerOptions,
RequestVideoEventScheduler,
RequestVideoEventSchedulerOptions
2017-05-15 22:22:03 +02:00
} from './request'
2017-06-16 10:36:18 +02:00
import {
PodInstance,
VideoInstance
} from '../models'
import {
RequestEndpoint,
RequestVideoEventType,
RequestVideoQaduType,
RemoteVideoCreateData,
RemoteVideoUpdateData,
RemoteVideoRemoveData,
2017-07-11 10:59:13 +02:00
RemoteVideoReportAbuseData,
ResultList,
2017-08-25 11:45:31 +02:00
Pod as FormattedPod
2017-06-16 10:36:18 +02:00
} from '../../shared'
2017-06-10 22:15:25 +02:00
type QaduParam = { videoId: number, type: RequestVideoQaduType }
type EventParam = { videoId: number, type: RequestVideoEventType }
2017-05-15 22:22:03 +02:00
const ENDPOINT_ACTIONS = REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS]
const requestScheduler = new RequestScheduler()
2017-02-27 21:56:55 +01:00
const requestVideoQaduScheduler = new RequestVideoQaduScheduler()
const requestVideoEventScheduler = new RequestVideoEventScheduler()
2017-05-15 22:22:03 +02:00
function activateSchedulers () {
requestScheduler.activate()
2017-02-27 21:56:55 +01:00
requestVideoQaduScheduler.activate()
requestVideoEventScheduler.activate()
}
function addVideoToFriends (videoData: RemoteVideoCreateData, transaction: Sequelize.Transaction) {
const options = {
type: ENDPOINT_ACTIONS.ADD,
2017-05-15 22:22:03 +02:00
endpoint: REQUEST_ENDPOINTS.VIDEOS,
data: videoData,
transaction
}
return createRequest(options)
}
function updateVideoToFriends (videoData: RemoteVideoUpdateData, transaction: Sequelize.Transaction) {
const options = {
type: ENDPOINT_ACTIONS.UPDATE,
2017-05-15 22:22:03 +02:00
endpoint: REQUEST_ENDPOINTS.VIDEOS,
data: videoData,
transaction
}
return createRequest(options)
2017-01-04 20:59:23 +01:00
}
function removeVideoToFriends (videoParams: RemoteVideoRemoveData) {
const options = {
type: ENDPOINT_ACTIONS.REMOVE,
2017-05-15 22:22:03 +02:00
endpoint: REQUEST_ENDPOINTS.VIDEOS,
2017-06-10 22:15:25 +02:00
data: videoParams,
transaction: null
}
return createRequest(options)
2017-01-04 20:59:23 +01:00
}
function reportAbuseVideoToFriend (reportData: RemoteVideoReportAbuseData, video: VideoInstance, transaction: Sequelize.Transaction) {
2017-01-10 22:24:42 +01:00
const options = {
type: ENDPOINT_ACTIONS.REPORT_ABUSE,
2017-05-15 22:22:03 +02:00
endpoint: REQUEST_ENDPOINTS.VIDEOS,
2017-01-10 22:24:42 +01:00
data: reportData,
2017-06-10 22:15:25 +02:00
toIds: [ video.Author.podId ],
transaction
2017-01-10 22:24:42 +01:00
}
return createRequest(options)
2016-12-29 19:07:05 +01:00
}
function quickAndDirtyUpdateVideoToFriends (qaduParam: QaduParam, transaction?: Sequelize.Transaction) {
const options = {
2017-06-10 22:15:25 +02:00
videoId: qaduParam.videoId,
type: qaduParam.type,
transaction
}
return createVideoQaduRequest(options)
}
function quickAndDirtyUpdatesVideoToFriends (qadusParams: QaduParam[], transaction: Sequelize.Transaction) {
2017-03-08 21:35:43 +01:00
const tasks = []
2017-07-11 17:04:57 +02:00
qadusParams.forEach(qaduParams => {
tasks.push(quickAndDirtyUpdateVideoToFriends(qaduParams, transaction))
2017-03-08 21:35:43 +01:00
})
return Promise.all(tasks)
2017-03-08 21:35:43 +01:00
}
function addEventToRemoteVideo (eventParam: EventParam, transaction?: Sequelize.Transaction) {
2017-02-26 18:57:33 +01:00
const options = {
2017-06-10 22:15:25 +02:00
videoId: eventParam.videoId,
type: eventParam.type,
2017-02-26 18:57:33 +01:00
transaction
}
return createVideoEventRequest(options)
2017-02-26 18:57:33 +01:00
}
function addEventsToRemoteVideo (eventsParams: EventParam[], transaction: Sequelize.Transaction) {
2017-03-08 21:35:43 +01:00
const tasks = []
2017-07-11 17:04:57 +02:00
eventsParams.forEach(eventParams => {
tasks.push(addEventToRemoteVideo(eventParams, transaction))
2017-03-08 21:35:43 +01:00
})
return Promise.all(tasks)
2017-03-08 21:35:43 +01:00
}
function hasFriends () {
return db.Pod.countAll().then(count => count !== 0)
}
function makeFriends (hosts: string[]) {
const podsScore = {}
logger.info('Make friends!')
return getMyPublicCert()
.then(cert => {
2017-07-07 16:57:28 +02:00
return Promise.each(hosts, host => computeForeignPodsList(host, podsScore)).then(() => cert)
})
.then(cert => {
logger.debug('Pods scores computed.', { podsScore: podsScore })
const podsList = computeWinningPods(hosts, podsScore)
logger.debug('Pods that we keep.', { podsToKeep: podsList })
return makeRequestsToWinningPods(cert, podsList)
2016-02-04 21:10:33 +01:00
})
}
function quitFriends () {
// Stop pool requests
requestScheduler.deactivate()
return requestScheduler.flush()
.then(() => {
return requestVideoQaduScheduler.flush()
})
.then(() => {
return db.Pod.list()
})
.then(pods => {
const requestParams = {
2017-06-10 22:15:25 +02:00
method: 'POST' as 'POST',
2017-05-15 22:22:03 +02:00
path: '/api/' + API_VERSION + '/remote/pods/remove',
toPod: null
2016-02-04 21:10:33 +01:00
}
// Announce we quit them
// We don't care if the request fails
// The other pod will exclude us automatically after a while
return Promise.map(pods, pod => {
requestParams.toPod = pod
2017-07-07 16:57:28 +02:00
return makeSecureRequest(requestParams)
}, { concurrency: REQUESTS_IN_PARALLEL })
.then(() => pods)
.catch(err => {
2017-07-07 18:26:12 +02:00
logger.error('Some errors while quitting friends.', err)
// Don't stop the process
})
})
.then(pods => {
const tasks = []
pods.forEach(pod => tasks.push(pod.destroy()))
return Promise.all(pods)
})
.then(() => {
logger.info('Removed all remote videos.')
// Don't forget to re activate the scheduler, even if there was an error
return requestScheduler.activate()
})
.finally(() => requestScheduler.activate())
}
2016-02-04 21:10:33 +01:00
2017-06-10 22:15:25 +02:00
function sendOwnedVideosToPod (podId: number) {
db.Video.listOwnedAndPopulateAuthorAndTags()
.then(videosList => {
const tasks = []
videosList.forEach(video => {
const promise = video.toAddRemoteJSON()
.then(remoteVideo => {
const options = {
type: 'add',
endpoint: REQUEST_ENDPOINTS.VIDEOS,
data: remoteVideo,
toIds: [ podId ],
transaction: null
}
return createRequest(options)
})
.catch(err => {
2017-07-07 18:26:12 +02:00
logger.error('Cannot convert video to remote.', err)
// Don't break the process
return undefined
})
tasks.push(promise)
})
return Promise.all(tasks)
})
}
2016-02-04 21:10:33 +01:00
2017-07-12 11:56:02 +02:00
function fetchRemotePreview (pod: PodInstance, video: VideoInstance) {
const host = video.Author.Pod.host
const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
}
function removeFriend (pod: PodInstance) {
const requestParams = {
method: 'POST' as 'POST',
path: '/api/' + API_VERSION + '/remote/pods/remove',
toPod: pod
}
return makeSecureRequest(requestParams)
.then(() => pod.destroy())
.then(() => {
logger.info('Removed friend.')
})
.catch(err => {
logger.error('Some errors while quitting friend %s (id: %d).', pod.host, pod.id, err)
})
}
2017-02-27 21:56:55 +01:00
function getRequestScheduler () {
return requestScheduler
}
function getRequestVideoQaduScheduler () {
return requestVideoQaduScheduler
}
function getRequestVideoEventScheduler () {
return requestVideoEventScheduler
}
// ---------------------------------------------------------------------------
2016-02-04 21:10:33 +01:00
2017-05-15 22:22:03 +02:00
export {
activateSchedulers,
addVideoToFriends,
updateVideoToFriends,
reportAbuseVideoToFriend,
quickAndDirtyUpdateVideoToFriends,
quickAndDirtyUpdatesVideoToFriends,
addEventToRemoteVideo,
addEventsToRemoteVideo,
hasFriends,
makeFriends,
quitFriends,
removeFriend,
2017-05-15 22:22:03 +02:00
removeVideoToFriends,
sendOwnedVideosToPod,
getRequestScheduler,
getRequestVideoQaduScheduler,
2017-07-12 11:56:02 +02:00
getRequestVideoEventScheduler,
fetchRemotePreview
2017-05-15 22:22:03 +02:00
}
2016-02-04 21:10:33 +01:00
// ---------------------------------------------------------------------------
2016-02-04 21:10:33 +01:00
function computeForeignPodsList (host: string, podsScore: { [ host: string ]: number }) {
// TODO: type res
2017-07-11 10:59:13 +02:00
return getForeignPodsList(host).then(res => {
const foreignPodsList: { host: string }[] = res.data
// Let's give 1 point to the pod we ask the friends list
foreignPodsList.push({ host })
foreignPodsList.forEach(foreignPod => {
const foreignPodHost = foreignPod.host
if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++
else podsScore[foreignPodHost] = 1
})
return undefined
})
}
2017-06-10 22:15:25 +02:00
function computeWinningPods (hosts: string[], podsScore: { [ host: string ]: number }) {
// Build the list of pods to add
// Only add a pod if it exists in more than a half base pods
const podsList = []
const baseScore = hosts.length / 2
2017-01-17 21:17:07 +01:00
Object.keys(podsScore).forEach(podHost => {
// If the pod is not me and with a good score we add it
if (isMe(podHost) === false && podsScore[podHost] > baseScore) {
podsList.push({ host: podHost })
}
})
return podsList
}
function getForeignPodsList (host: string) {
2017-08-25 11:45:31 +02:00
return new Promise< ResultList<FormattedPod> >((res, rej) => {
const path = '/api/' + API_VERSION + '/pods'
2016-02-04 21:10:33 +01:00
2017-07-11 17:04:57 +02:00
request.get(REMOTE_SCHEME.HTTP + '://' + host + path, (err, response, body) => {
if (err) return rej(err)
2016-02-05 19:02:05 +01:00
try {
const json = JSON.parse(body)
return res(json)
} catch (err) {
return rej(err)
}
})
})
}
function makeRequestsToWinningPods (cert: string, podsList: PodInstance[]) {
// Stop pool requests
requestScheduler.deactivate()
// Flush pool requests
requestScheduler.forceSend()
return Promise.map(podsList, pod => {
const params = {
2017-05-15 22:22:03 +02:00
url: REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + API_VERSION + '/pods/',
2017-06-10 22:15:25 +02:00
method: 'POST' as 'POST',
json: {
2017-05-15 22:22:03 +02:00
host: CONFIG.WEBSERVER.HOST,
email: CONFIG.ADMIN.EMAIL,
publicKey: cert
}
}
return makeRetryRequest(params)
.then(({ response, body }) => {
body = body as { cert: string, email: string }
if (response.statusCode === 200) {
const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email })
return podObj.save()
.then(podCreated => {
// Add our videos to the request scheduler
sendOwnedVideosToPod(podCreated.id)
})
.catch(err => {
2017-07-07 18:26:12 +02:00
logger.error('Cannot add friend %s pod.', pod.host, err)
})
} else {
logger.error('Status not 200 for %s pod.', pod.host)
}
})
.catch(err => {
logger.error('Error with adding %s pod.', pod.host, { error: err.stack })
// Don't break the process
})
}, { concurrency: REQUESTS_IN_PARALLEL })
.then(() => logger.debug('makeRequestsToWinningPods finished.'))
.finally(() => {
// Final callback, we've ended all the requests
// Now we made new friends, we can re activate the pool of requests
requestScheduler.activate()
})
}
2017-01-04 20:59:23 +01:00
// Wrapper that populate "toIds" argument with all our friends if it is not specified
2017-06-10 22:15:25 +02:00
type CreateRequestOptions = {
type: string
2017-06-16 10:36:18 +02:00
endpoint: RequestEndpoint
2017-06-10 22:15:25 +02:00
data: Object
toIds?: number[]
transaction: Sequelize.Transaction
}
function createRequest (options: CreateRequestOptions) {
if (options.toIds !== undefined) return requestScheduler.createRequest(options as RequestSchedulerOptions)
2016-12-11 21:50:51 +01:00
2017-01-04 20:59:23 +01:00
// If the "toIds" pods is not specified, we send the request to all our friends
return db.Pod.listAllIds(options.transaction).then(podIds => {
const newOptions = Object.assign(options, { toIds: podIds })
return requestScheduler.createRequest(newOptions)
})
}
function createVideoQaduRequest (options: RequestVideoQaduSchedulerOptions) {
return requestVideoQaduScheduler.createRequest(options)
}
function createVideoEventRequest (options: RequestVideoEventSchedulerOptions) {
return requestVideoEventScheduler.createRequest(options)
2017-02-26 18:57:33 +01:00
}
2017-06-10 22:15:25 +02:00
function isMe (host: string) {
2017-05-15 22:22:03 +02:00
return host === CONFIG.WEBSERVER.HOST
}