Fix CLI tools

pull/4271/head
Chocobozzz 2021-07-09 15:03:44 +02:00
parent 12edc1495a
commit 078f17e6d9
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
11 changed files with 190 additions and 227 deletions

View File

@ -12,6 +12,7 @@ import {
doubleFollow,
flushAndRunServer,
getLocalIdByUUID,
getMyUserInformation,
getVideo,
getVideosList,
ImportsCommand,
@ -52,6 +53,14 @@ describe('Test CLI wrapper', function () {
describe('Authentication and instance selection', function () {
it('Should get an access token', async function () {
const stdout = await cliCommand.execWithEnv(`${cmd} token --url ${server.url} --username user_1 --password super_password`)
const token = stdout.trim()
const res = await getMyUserInformation(server.url, token)
expect(res.body.username).to.equal('user_1')
})
it('Should display no selected instance', async function () {
this.timeout(60000)

View File

@ -1,6 +1,7 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import { expect } from 'chai'
import {
cleanupTests,
createUser,
@ -18,8 +19,6 @@ import {
} from '@shared/extra-utils'
import { VideoDetails } from '@shared/models'
const expect = chai.expect
describe('Test update host scripts', function () {
let server: ServerInfo

View File

@ -1,14 +1,14 @@
import { Command } from 'commander'
import { Netrc } from 'netrc-parser'
import { getAppNumber, isTestInstance } from '../helpers/core-utils'
import { join } from 'path'
import { root } from '../../shared/extra-utils/miscs/miscs'
import { getVideoChannel } from '../../shared/extra-utils/videos/video-channels'
import { VideoChannel, VideoPrivacy } from '../../shared/models/videos'
import { createLogger, format, transports } from 'winston'
import { assignCommands, ServerInfo } from '@shared/extra-utils'
import { getAccessToken } from '@shared/extra-utils/users/login'
import { getMyUserInformation } from '@shared/extra-utils/users/users'
import { User, UserRole } from '@shared/models'
import { getAccessToken } from '@shared/extra-utils/users/login'
import { Command } from 'commander'
import { root } from '../../shared/extra-utils/miscs/miscs'
import { VideoPrivacy } from '../../shared/models/videos'
import { getAppNumber, isTestInstance } from '../helpers/core-utils'
let configName = 'PeerTube/CLI'
if (isTestInstance()) configName += `-${getAppNumber()}`
@ -30,6 +30,10 @@ async function getAdminTokenOrDie (url: string, username: string, password: stri
return accessToken
}
async function getAccessTokenOrDie (url: string, username: string, password: string) {
return getAccessToken(url, username, password)
}
interface Settings {
remotes: any[]
default: number
@ -128,7 +132,7 @@ function buildCommonVideoOptions (command: Command) {
.option('-v, --verbose <verbose>', 'Verbosity, from 0/\'error\' to 4/\'debug\'', 'info')
}
async function buildVideoAttributesFromCommander (url: string, command: Command, defaultAttributes: any = {}) {
async function buildVideoAttributesFromCommander (server: ServerInfo, command: Command, defaultAttributes: any = {}) {
const options = command.opts()
const defaultBooleanAttributes = {
@ -164,8 +168,7 @@ async function buildVideoAttributesFromCommander (url: string, command: Command,
Object.assign(videoAttributes, booleanAttributes)
if (options.channelName) {
const res = await getVideoChannel(url, options.channelName)
const videoChannel: VideoChannel = res.body
const videoChannel = await server.channelsCommand.get({ channelName: options.channelName })
Object.assign(videoAttributes, { channelId: videoChannel.id })
@ -184,6 +187,13 @@ function getServerCredentials (program: Command) {
})
}
function buildServer (url: string, accessToken?: string): ServerInfo {
const server = { url, accessToken, internalServerNumber: undefined }
assignCommands(server)
return server
}
function getLogger (logLevel = 'info') {
const logLevels = {
0: 0,
@ -230,5 +240,7 @@ export {
buildCommonVideoOptions,
buildVideoAttributesFromCommander,
getAdminTokenOrDie
getAdminTokenOrDie,
getAccessTokenOrDie,
buildServer
}

View File

@ -2,7 +2,7 @@ import { registerTSPaths } from '../helpers/register-ts-paths'
registerTSPaths()
import { program } from 'commander'
import { getClient, Server, serverLogin } from '../../shared/extra-utils'
import { getAccessToken } from '../../shared/extra-utils'
program
.option('-u, --url <url>', 'Server url')
@ -24,22 +24,7 @@ if (
process.exit(-1)
}
getClient(options.url)
.then(res => {
const server = {
url: options.url,
user: {
username: options.username,
password: options.password
},
client: {
id: res.body.client_id,
secret: res.body.client_secret
}
} as Server
return serverLogin(server)
})
getAccessToken(options.url, options.username, options.password)
.then(accessToken => {
console.log(accessToken)
process.exit(0)

View File

@ -8,17 +8,19 @@ import { truncate } from 'lodash'
import { join } from 'path'
import * as prompt from 'prompt'
import { promisify } from 'util'
import { advancedVideosSearch, getClient, getVideoCategories, login, uploadVideo } from '../../shared/extra-utils/index'
import { YoutubeDL } from '@server/helpers/youtube-dl'
import { getVideoCategories, uploadVideo } from '../../shared/extra-utils/index'
import { sha256 } from '../helpers/core-utils'
import { doRequestAndSaveToFile } from '../helpers/requests'
import { CONSTRAINTS_FIELDS } from '../initializers/constants'
import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getLogger, getServerCredentials } from './cli'
import { YoutubeDL } from '@server/helpers/youtube-dl'
type UserInfo = {
username: string
password: string
}
import {
buildCommonVideoOptions,
buildServer,
buildVideoAttributesFromCommander,
getAccessTokenOrDie,
getLogger,
getServerCredentials
} from './cli'
const processOptions = {
maxBuffer: Infinity
@ -62,17 +64,13 @@ getServerCredentials(command)
url = normalizeTargetUrl(url)
options.targetUrl = normalizeTargetUrl(options.targetUrl)
const user = { username, password }
run(url, user)
run(url, username, password)
.catch(err => exitError(err))
})
.catch(err => console.error(err))
async function run (url: string, user: UserInfo) {
if (!user.password) {
user.password = await promptPassword()
}
async function run (url: string, username: string, password: string) {
if (!password) password = await promptPassword()
const youtubeDLBinary = await YoutubeDL.safeGetYoutubeDL()
@ -111,7 +109,8 @@ async function run (url: string, user: UserInfo) {
await processVideo({
cwd: options.tmpdir,
url,
user,
username,
password,
youtubeInfo: info
})
} catch (err) {
@ -119,17 +118,18 @@ async function run (url: string, user: UserInfo) {
}
}
log.info('Video/s for user %s imported: %s', user.username, options.targetUrl)
log.info('Video/s for user %s imported: %s', username, options.targetUrl)
process.exit(0)
}
async function processVideo (parameters: {
cwd: string
url: string
user: { username: string, password: string }
username: string
password: string
youtubeInfo: any
}) {
const { youtubeInfo, cwd, url, user } = parameters
const { youtubeInfo, cwd, url, username, password } = parameters
const youtubeDL = new YoutubeDL('', [])
log.debug('Fetching object.', youtubeInfo)
@ -138,22 +138,29 @@ async function processVideo (parameters: {
log.debug('Fetched object.', videoInfo)
const originallyPublishedAt = youtubeDL.buildOriginallyPublishedAt(videoInfo)
if (options.since && originallyPublishedAt && originallyPublishedAt.getTime() < options.since.getTime()) {
log.info('Video "%s" has been published before "%s", don\'t upload it.\n',
videoInfo.title, formatDate(options.since))
return
}
if (options.until && originallyPublishedAt && originallyPublishedAt.getTime() > options.until.getTime()) {
log.info('Video "%s" has been published after "%s", don\'t upload it.\n',
videoInfo.title, formatDate(options.until))
log.info('Video "%s" has been published before "%s", don\'t upload it.\n', videoInfo.title, formatDate(options.since))
return
}
const result = await advancedVideosSearch(url, { search: videoInfo.title, sort: '-match', searchTarget: 'local' })
if (options.until && originallyPublishedAt && originallyPublishedAt.getTime() > options.until.getTime()) {
log.info('Video "%s" has been published after "%s", don\'t upload it.\n', videoInfo.title, formatDate(options.until))
return
}
const server = buildServer(url)
const { data } = await server.searchCommand.advancedVideoSearch({
search: {
search: videoInfo.title,
sort: '-match',
searchTarget: 'local'
}
})
log.info('############################################################\n')
if (result.body.data.find(v => v.name === videoInfo.title)) {
if (data.find(v => v.name === videoInfo.title)) {
log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.title)
return
}
@ -172,7 +179,8 @@ async function processVideo (parameters: {
youtubeDL,
cwd,
url,
user,
username,
password,
videoInfo: normalizeObject(videoInfo),
videoPath: path
})
@ -187,9 +195,10 @@ async function uploadVideoOnPeerTube (parameters: {
videoPath: string
cwd: string
url: string
user: { username: string, password: string }
username: string
password: string
}) {
const { youtubeDL, videoInfo, videoPath, cwd, url, user } = parameters
const { youtubeDL, videoInfo, videoPath, cwd, url, username, password } = parameters
const category = await getCategory(videoInfo.categories, url)
const licence = getLicence(videoInfo.license)
@ -223,7 +232,10 @@ async function uploadVideoOnPeerTube (parameters: {
tags
}
const videoAttributes = await buildVideoAttributesFromCommander(url, program, defaultAttributes)
let accessToken = await getAccessTokenOrDie(url, username, password)
const server = buildServer(url, accessToken)
const videoAttributes = await buildVideoAttributesFromCommander(server, program, defaultAttributes)
Object.assign(videoAttributes, {
originallyPublishedAt: originallyPublishedAt ? originallyPublishedAt.toISOString() : null,
@ -234,15 +246,13 @@ async function uploadVideoOnPeerTube (parameters: {
log.info('\nUploading on PeerTube video "%s".', videoAttributes.name)
let accessToken = await getAccessTokenOrDie(url, user)
try {
await uploadVideo(url, accessToken, videoAttributes)
} catch (err) {
if (err.message.indexOf('401') !== -1) {
log.info('Got 401 Unauthorized, token may have expired, renewing token and retry.')
accessToken = await getAccessTokenOrDie(url, user)
accessToken = await getAccessTokenOrDie(url, username, password)
await uploadVideo(url, accessToken, videoAttributes)
} else {
@ -362,21 +372,6 @@ async function promptPassword () {
})
}
async function getAccessTokenOrDie (url: string, user: UserInfo) {
const resClient = await getClient(url)
const client = {
id: resClient.body.client_id,
secret: resClient.body.client_secret
}
try {
const res = await login(url, client, user)
return res.body.access_token
} catch (err) {
exitError('Cannot authenticate. Please check your username/password.')
}
}
function parseDate (dateAsStr: string): Date {
if (!/\d{4}-\d{2}-\d{2}/.test(dateAsStr)) {
exitError(`Invalid date passed: ${dateAsStr}. Expected format: YYYY-MM-DD. See help for usage.`)

View File

@ -4,9 +4,8 @@ import { registerTSPaths } from '../helpers/register-ts-paths'
registerTSPaths()
import { program, Command, OptionValues } from 'commander'
import { installPlugin, listPlugins, uninstallPlugin, updatePlugin } from '../../shared/extra-utils/server/plugins'
import { getAdminTokenOrDie, getServerCredentials } from './cli'
import { PeerTubePlugin, PluginType } from '../../shared/models'
import { buildServer, getAdminTokenOrDie, getServerCredentials } from './cli'
import { PluginType } from '../../shared/models'
import { isAbsolute } from 'path'
import * as CliTable3 from 'cli-table3'
@ -63,28 +62,21 @@ program.parse(process.argv)
async function pluginsListCLI (command: Command, options: OptionValues) {
const { url, username, password } = await getServerCredentials(command)
const accessToken = await getAdminTokenOrDie(url, username, password)
const token = await getAdminTokenOrDie(url, username, password)
const server = buildServer(url, token)
let pluginType: PluginType
if (options.onlyThemes) pluginType = PluginType.THEME
if (options.onlyPlugins) pluginType = PluginType.PLUGIN
const res = await listPlugins({
url,
accessToken,
start: 0,
count: 100,
sort: 'name',
pluginType
})
const plugins: PeerTubePlugin[] = res.body.data
const { data } = await server.pluginsCommand.list({ start: 0, count: 100, sort: 'name', pluginType })
const table = new CliTable3({
head: [ 'name', 'version', 'homepage' ],
colWidths: [ 50, 10, 50 ]
}) as any
for (const plugin of plugins) {
for (const plugin of data) {
const npmName = plugin.type === PluginType.PLUGIN
? 'peertube-plugin-' + plugin.name
: 'peertube-theme-' + plugin.name
@ -113,15 +105,11 @@ async function installPluginCLI (command: Command, options: OptionValues) {
}
const { url, username, password } = await getServerCredentials(command)
const accessToken = await getAdminTokenOrDie(url, username, password)
const token = await getAdminTokenOrDie(url, username, password)
const server = buildServer(url, token)
try {
await installPlugin({
url,
accessToken,
npmName: options.npmName,
path: options.path
})
await server.pluginsCommand.install({ npmName: options.npmName, path: options.path })
} catch (err) {
console.error('Cannot install plugin.', err)
process.exit(-1)
@ -144,15 +132,11 @@ async function updatePluginCLI (command: Command, options: OptionValues) {
}
const { url, username, password } = await getServerCredentials(command)
const accessToken = await getAdminTokenOrDie(url, username, password)
const token = await getAdminTokenOrDie(url, username, password)
const server = buildServer(url, token)
try {
await updatePlugin({
url,
accessToken,
npmName: options.npmName,
path: options.path
})
await server.pluginsCommand.update({ npmName: options.npmName, path: options.path })
} catch (err) {
console.error('Cannot update plugin.', err)
process.exit(-1)
@ -170,14 +154,11 @@ async function uninstallPluginCLI (command: Command, options: OptionValues) {
}
const { url, username, password } = await getServerCredentials(command)
const accessToken = await getAdminTokenOrDie(url, username, password)
const token = await getAdminTokenOrDie(url, username, password)
const server = buildServer(url, token)
try {
await uninstallPlugin({
url,
accessToken,
npmName: options.npmName
})
await server.pluginsCommand.uninstall({ npmName: options.npmName })
} catch (err) {
console.error('Cannot uninstall plugin.', err)
process.exit(-1)

View File

@ -1,17 +1,14 @@
// eslint-disable @typescript-eslint/no-unnecessary-type-assertion
import { registerTSPaths } from '../helpers/register-ts-paths'
registerTSPaths()
import { program, Command } from 'commander'
import { getAdminTokenOrDie, getServerCredentials } from './cli'
import { VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy } from '@shared/extra-utils/server/redundancy'
import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
import validator from 'validator'
import * as CliTable3 from 'cli-table3'
import { URL } from 'url'
import { Command, program } from 'commander'
import { uniq } from 'lodash'
import { URL } from 'url'
import validator from 'validator'
import { HttpStatusCode } from '@shared/core-utils/miscs/http-error-codes'
import { VideoRedundanciesTarget } from '@shared/models'
import { buildServer, getAdminTokenOrDie, getServerCredentials } from './cli'
import bytes = require('bytes')
@ -63,15 +60,16 @@ program.parse(process.argv)
async function listRedundanciesCLI (target: VideoRedundanciesTarget) {
const { url, username, password } = await getServerCredentials(program)
const accessToken = await getAdminTokenOrDie(url, username, password)
const token = await getAdminTokenOrDie(url, username, password)
const server = buildServer(url, token)
const redundancies = await listVideoRedundanciesData(url, accessToken, target)
const { data } = await server.redundancyCommand.listVideos({ start: 0, count: 100, sort: 'name', target })
const table = new CliTable3({
head: [ 'video id', 'video name', 'video url', 'files', 'playlists', 'by instances', 'total size' ]
}) as any
for (const redundancy of redundancies) {
for (const redundancy of data) {
const webtorrentFiles = redundancy.redundancies.files
const streamingPlaylists = redundancy.redundancies.streamingPlaylists
@ -106,7 +104,8 @@ async function listRedundanciesCLI (target: VideoRedundanciesTarget) {
async function addRedundancyCLI (options: { video: number }, command: Command) {
const { url, username, password } = await getServerCredentials(command)
const accessToken = await getAdminTokenOrDie(url, username, password)
const token = await getAdminTokenOrDie(url, username, password)
const server = buildServer(url, token)
if (!options.video || validator.isInt('' + options.video) === false) {
console.error('You need to specify the video id to duplicate and it should be a number.\n')
@ -115,11 +114,7 @@ async function addRedundancyCLI (options: { video: number }, command: Command) {
}
try {
await addVideoRedundancy({
url,
accessToken,
videoId: options.video
})
await server.redundancyCommand.addVideo({ videoId: options.video })
console.log('Video will be duplicated by your instance!')
@ -139,7 +134,8 @@ async function addRedundancyCLI (options: { video: number }, command: Command) {
async function removeRedundancyCLI (options: { video: number }, command: Command) {
const { url, username, password } = await getServerCredentials(command)
const accessToken = await getAdminTokenOrDie(url, username, password)
const token = await getAdminTokenOrDie(url, username, password)
const server = buildServer(url, token)
if (!options.video || validator.isInt('' + options.video) === false) {
console.error('You need to specify the video id to remove from your redundancies.\n')
@ -149,12 +145,12 @@ async function removeRedundancyCLI (options: { video: number }, command: Command
const videoId = parseInt(options.video + '', 10)
let redundancies = await listVideoRedundanciesData(url, accessToken, 'my-videos')
let videoRedundancy = redundancies.find(r => videoId === r.id)
const myVideoRedundancies = await server.redundancyCommand.listVideos({ target: 'my-videos' })
let videoRedundancy = myVideoRedundancies.data.find(r => videoId === r.id)
if (!videoRedundancy) {
redundancies = await listVideoRedundanciesData(url, accessToken, 'remote-videos')
videoRedundancy = redundancies.find(r => videoId === r.id)
const remoteVideoRedundancies = await server.redundancyCommand.listVideos({ target: 'remote-videos' })
videoRedundancy = remoteVideoRedundancies.data.find(r => videoId === r.id)
}
if (!videoRedundancy) {
@ -168,11 +164,7 @@ async function removeRedundancyCLI (options: { video: number }, command: Command
.map(r => r.id)
for (const id of ids) {
await removeVideoRedundancy({
url,
accessToken,
redundancyId: id
})
await server.redundancyCommand.removeVideo({ redundancyId: id })
}
console.log('Video redundancy removed!')
@ -183,16 +175,3 @@ async function removeRedundancyCLI (options: { video: number }, command: Command
process.exit(-1)
}
}
async function listVideoRedundanciesData (url: string, accessToken: string, target: VideoRedundanciesTarget) {
const res = await listVideoRedundancies({
url,
accessToken,
start: 0,
count: 100,
sort: 'name',
target
})
return res.body.data as VideoRedundancy[]
}

View File

@ -6,7 +6,7 @@ import { access, constants } from 'fs-extra'
import { isAbsolute } from 'path'
import { getAccessToken } from '../../shared/extra-utils'
import { uploadVideo } from '../../shared/extra-utils/'
import { buildCommonVideoOptions, buildVideoAttributesFromCommander, getServerCredentials } from './cli'
import { buildCommonVideoOptions, buildServer, buildVideoAttributesFromCommander, getServerCredentials } from './cli'
let command = program
.name('upload')
@ -46,13 +46,14 @@ getServerCredentials(command)
.catch(err => console.error(err))
async function run (url: string, username: string, password: string) {
const accessToken = await getAccessToken(url, username, password)
const token = await getAccessToken(url, username, password)
const server = buildServer(url, token)
await access(options.file, constants.F_OK)
console.log('Uploading %s video...', options.videoName)
const videoAttributes = await buildVideoAttributesFromCommander(url, program)
const videoAttributes = await buildVideoAttributesFromCommander(server, program)
Object.assign(videoAttributes, {
fixture: options.file,
@ -61,7 +62,7 @@ async function run (url: string, username: string, password: string) {
})
try {
await uploadVideo(url, accessToken, videoAttributes)
await uploadVideo(url, token, videoAttributes)
console.log(`Video ${options.videoName} uploaded.`)
process.exit(0)
} catch (err) {

View File

@ -1,26 +1,23 @@
import { registerTSPaths } from '../helpers/register-ts-paths'
registerTSPaths()
import { LiveVideo, LiveVideoCreate, VideoPrivacy } from '@shared/models'
import { program } from 'commander'
import { LiveVideoCreate, VideoPrivacy } from '@shared/models'
import {
createLive,
flushAndRunServer,
getLive,
killallServers,
sendRTMPStream,
ServerInfo,
setAccessTokensToServers,
setDefaultVideoChannel,
updateCustomSubConfig
setDefaultVideoChannel
} from '../../shared/extra-utils'
import { registerTSPaths } from '../helpers/register-ts-paths'
registerTSPaths()
type CommandType = 'live-mux' | 'live-transcoding'
registerTSPaths()
const command = program
.name('test')
.name('test-live')
.option('-t, --type <type>', 'live-muxing|live-transcoding')
.parse(process.argv)
@ -63,11 +60,9 @@ async function run () {
console.log('Creating live.')
const res = await createLive(server.url, server.accessToken, attributes)
const liveVideoUUID = res.body.video.uuid
const { uuid: liveVideoUUID } = await server.liveCommand.create({ fields: attributes })
const resLive = await getLive(server.url, server.accessToken, liveVideoUUID)
const live: LiveVideo = resLive.body
const live = await server.liveCommand.get({ videoId: liveVideoUUID })
console.log('Sending RTMP stream.')
@ -87,7 +82,8 @@ async function run () {
// ----------------------------------------------------------------------------
async function buildConfig (server: ServerInfo, commandType: CommandType) {
await updateCustomSubConfig(server.url, server.accessToken, {
await server.configCommand.updateCustomSubConfig({
newConfig: {
instance: {
customizations: {
javascript: '',
@ -101,5 +97,6 @@ async function buildConfig (server: ServerInfo, commandType: CommandType) {
enabled: commandType === 'live-transcoding'
}
}
}
})
}

View File

@ -41,25 +41,25 @@ import { RedundancyCommand } from './redundancy-command'
import { StatsCommand } from './stats-command'
interface ServerInfo {
app: ChildProcess
app?: ChildProcess
url: string
host: string
hostname: string
port: number
host?: string
hostname?: string
port?: number
rtmpPort: number
rtmpPort?: number
parallel: boolean
parallel?: boolean
internalServerNumber: number
serverNumber: number
serverNumber?: number
client: {
id: string
secret: string
client?: {
id?: string
secret?: string
}
user: {
user?: {
username: string
password: string
email?: string
@ -328,6 +328,14 @@ async function runServer (server: ServerInfo, configOverrideArg?: any, args = []
} catch { /* empty */ }
})
assignCommands(server)
res(server)
})
})
}
function assignCommands (server: ServerInfo) {
server.bulkCommand = new BulkCommand(server)
server.cliCommand = new CLICommand(server)
server.customPageCommand = new CustomPagesCommand(server)
@ -359,10 +367,6 @@ async function runServer (server: ServerInfo, configOverrideArg?: any, args = []
server.streamingPlaylistsCommand = new StreamingPlaylistsCommand(server)
server.channelsCommand = new ChannelsCommand(server)
server.commentsCommand = new CommentsCommand(server)
res(server)
})
})
}
async function reRunServer (server: ServerInfo, configOverride?: any) {
@ -475,5 +479,6 @@ export {
flushAndRunServer,
killallServers,
reRunServer,
assignCommands,
waitUntilLog
}

View File

@ -4,9 +4,9 @@ import { ServerInfo } from '../server/servers'
import { getClient } from '../server/clients'
import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
type Client = { id: string, secret: string }
type Client = { id?: string, secret?: string }
type User = { username: string, password: string }
type Server = { url: string, client: Client, user: User }
type Server = { url?: string, client?: Client, user?: User }
function login (url: string, client: Client, user: User, expectedStatus = HttpStatusCode.OK_200) {
const path = '/api/v1/users/token'