Add ability to list jobs

pull/143/head
Chocobozzz 2017-11-30 10:51:13 +01:00
parent 1f3e9feca2
commit 5cd8054542
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
29 changed files with 462 additions and 27 deletions

View File

@ -8,6 +8,7 @@ import { FollowsRoutes } from './follows'
import { UsersRoutes } from './users'
import { VideoAbusesRoutes } from './video-abuses'
import { VideoBlacklistRoutes } from './video-blacklist'
import { JobsRoutes } from './jobs/job.routes'
const adminRoutes: Routes = [
{
@ -24,7 +25,8 @@ const adminRoutes: Routes = [
...FollowsRoutes,
...UsersRoutes,
...VideoAbusesRoutes,
...VideoBlacklistRoutes
...VideoBlacklistRoutes,
...JobsRoutes
]
}
]

View File

@ -5,6 +5,9 @@ import { AdminRoutingModule } from './admin-routing.module'
import { AdminComponent } from './admin.component'
import { FollowersListComponent, FollowingAddComponent, FollowsComponent, FollowService } from './follows'
import { FollowingListComponent } from './follows/following-list/following-list.component'
import { JobsComponent } from './jobs/job.component'
import { JobsListComponent } from './jobs/jobs-list/jobs-list.component'
import { JobService } from './jobs/shared/job.service'
import { UserAddComponent, UserListComponent, UsersComponent, UserService, UserUpdateComponent } from './users'
import { VideoAbuseListComponent, VideoAbusesComponent } from './video-abuses'
import { VideoBlacklistComponent, VideoBlacklistListComponent } from './video-blacklist'
@ -33,7 +36,10 @@ import { VideoBlacklistComponent, VideoBlacklistListComponent } from './video-bl
VideoBlacklistListComponent,
VideoAbusesComponent,
VideoAbuseListComponent
VideoAbuseListComponent,
JobsComponent,
JobsListComponent
],
exports: [
@ -42,7 +48,8 @@ import { VideoBlacklistComponent, VideoBlacklistListComponent } from './video-bl
providers: [
FollowService,
UserService
UserService,
JobService
]
})
export class AdminModule { }

View File

@ -0,0 +1 @@
export * from './'

View File

@ -0,0 +1,6 @@
import { Component } from '@angular/core'
@Component({
template: '<router-outlet></router-outlet>'
})
export class JobsComponent {}

View File

@ -0,0 +1,35 @@
import { Routes } from '@angular/router'
import { UserRightGuard } from '../../core'
import { FollowingAddComponent } from './following-add'
import { UserRight } from '../../../../../shared'
import { FollowingListComponent } from './following-list/following-list.component'
import { JobsComponent } from './job.component'
import { JobsListComponent } from './jobs-list/jobs-list.component'
export const JobsRoutes: Routes = [
{
path: 'jobs',
component: JobsComponent,
canActivate: [ UserRightGuard ],
data: {
userRight: UserRight.MANAGE_JOBS
},
children: [
{
path: '',
redirectTo: 'list',
pathMatch: 'full'
},
{
path: 'list',
component: JobsListComponent,
data: {
meta: {
title: 'Jobs list'
}
}
}
]
}
]

View File

@ -0,0 +1 @@
export * from './jobs-list.component'

View File

@ -0,0 +1,18 @@
<div class="row">
<div class="content-padding">
<h3>Jobs list</h3>
<p-dataTable
[value]="jobs" [lazy]="true" [paginator]="true" [totalRecords]="totalRecords" [rows]="rowsPerPage"
sortField="createdAt" (onLazyLoad)="loadLazy($event)"
>
<p-column field="id" header="ID"></p-column>
<p-column field="category" header="Category"></p-column>
<p-column field="handlerName" header="Handler name"></p-column>
<p-column field="handlerInputData" header="Input data"></p-column>
<p-column field="state" header="State"></p-column>
<p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
<p-column field="updatedAt" header="Updated date"></p-column>
</p-dataTable>
</div>
</div>

View File

@ -0,0 +1,50 @@
import { Component } from '@angular/core'
import { NotificationsService } from 'angular2-notifications'
import { SortMeta } from 'primeng/primeng'
import { Job } from '../../../../../../shared/index'
import { RestPagination, RestTable } from '../../../shared'
import { JobService } from '../shared'
import { RestExtractor } from '../../../shared/rest/rest-extractor.service'
@Component({
selector: 'my-jobs-list',
templateUrl: './jobs-list.component.html',
styleUrls: [ ]
})
export class JobsListComponent extends RestTable {
jobs: Job[] = []
totalRecords = 0
rowsPerPage = 10
sort: SortMeta = { field: 'createdAt', order: 1 }
pagination: RestPagination = { count: this.rowsPerPage, start: 0 }
constructor (
private notificationsService: NotificationsService,
private restExtractor: RestExtractor,
private jobsService: JobService
) {
super()
}
protected loadData () {
this.jobsService
.getJobs(this.pagination, this.sort)
.map(res => this.restExtractor.applyToResultListData(res, this.formatJob.bind(this)))
.subscribe(
resultList => {
this.jobs = resultList.data
this.totalRecords = resultList.total
},
err => this.notificationsService.error('Error', err.message)
)
}
private formatJob (job: Job) {
const handlerInputData = JSON.stringify(job.handlerInputData)
return Object.assign(job, {
handlerInputData
})
}
}

View File

@ -0,0 +1 @@
export * from './job.service'

View File

@ -0,0 +1,30 @@
import { HttpClient, HttpParams } from '@angular/common/http'
import { Injectable } from '@angular/core'
import { SortMeta } from 'primeng/primeng'
import 'rxjs/add/operator/catch'
import 'rxjs/add/operator/map'
import { Observable } from 'rxjs/Observable'
import { ResultList } from '../../../../../../shared'
import { Job } from '../../../../../../shared/models/job.model'
import { RestExtractor, RestPagination, RestService } from '../../../shared'
@Injectable()
export class JobService {
private static BASE_JOB_URL = API_URL + '/api/v1/jobs'
constructor (
private authHttp: HttpClient,
private restService: RestService,
private restExtractor: RestExtractor
) {}
getJobs (pagination: RestPagination, sort: SortMeta): Observable<ResultList<Job>> {
let params = new HttpParams()
params = this.restService.addRestGetParams(params, pagination, sort)
return this.authHttp.get<ResultList<Job>>(JobService.BASE_JOB_URL, { params })
.map(res => this.restExtractor.convertResultListDateToHuman(res))
.catch(err => this.restExtractor.handleError(err))
}
}

View File

@ -19,6 +19,11 @@
<span class="hidden-xs glyphicon glyphicon-eye-close"></span>
Video blacklist
</a>
<a *ngIf="hasJobsRight()" routerLink="/admin/jobs" routerLinkActive="active">
<span class="hidden-xs glyphicon glyphicon-tasks"></span>
Jobs
</a>
</div>
<div class="panel-block">

View File

@ -26,4 +26,8 @@ export class MenuAdminComponent {
hasVideoBlacklistRight () {
return this.auth.getUser().hasRight(UserRight.MANAGE_VIDEO_BLACKLIST)
}
hasJobsRight () {
return this.auth.getUser().hasRight(UserRight.MANAGE_JOBS)
}
}

View File

@ -7,6 +7,7 @@ import { configRouter } from './config'
import { serverRouter } from './server'
import { usersRouter } from './users'
import { videosRouter } from './videos'
import { jobsRouter } from './jobs'
const apiRouter = express.Router()
@ -15,6 +16,7 @@ apiRouter.use('/oauth-clients', oauthClientsRouter)
apiRouter.use('/config', configRouter)
apiRouter.use('/users', usersRouter)
apiRouter.use('/videos', videosRouter)
apiRouter.use('/jobs', jobsRouter)
apiRouter.use('/ping', pong)
apiRouter.use('/*', badRequest)

View File

@ -0,0 +1,34 @@
import * as express from 'express'
import { asyncMiddleware, jobsSortValidator, setJobsSort, setPagination } from '../../middlewares'
import { paginationValidator } from '../../middlewares/validators/pagination'
import { database as db } from '../../initializers'
import { getFormattedObjects } from '../../helpers/utils'
import { authenticate } from '../../middlewares/oauth'
import { ensureUserHasRight } from '../../middlewares/user-right'
import { UserRight } from '../../../shared/models/users/user-right.enum'
const jobsRouter = express.Router()
jobsRouter.get('/',
authenticate,
ensureUserHasRight(UserRight.MANAGE_JOBS),
paginationValidator,
jobsSortValidator,
setJobsSort,
setPagination,
asyncMiddleware(listJobs)
)
// ---------------------------------------------------------------------------
export {
jobsRouter
}
// ---------------------------------------------------------------------------
async function listJobs (req: express.Request, res: express.Response, next: express.NextFunction) {
const resultList = await db.Job.listForApi(req.query.start, req.query.count, req.query.sort)
return res.json(getFormattedObjects(resultList.data, resultList.total))
}

View File

@ -32,6 +32,7 @@ const SEARCHABLE_COLUMNS = {
// Sortable columns per schema
const SORTABLE_COLUMNS = {
USERS: [ 'id', 'username', 'createdAt' ],
JOBS: [ 'id', 'createdAt' ],
VIDEO_ABUSES: [ 'id', 'createdAt' ],
VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ],
VIDEOS: [ 'name', 'duration', 'createdAt', 'views', 'likes' ],

View File

@ -10,6 +10,12 @@ function setUsersSort (req: express.Request, res: express.Response, next: expres
return next()
}
function setJobsSort (req: express.Request, res: express.Response, next: express.NextFunction) {
if (!req.query.sort) req.query.sort = '-createdAt'
return next()
}
function setVideoAbusesSort (req: express.Request, res: express.Response, next: express.NextFunction) {
if (!req.query.sort) req.query.sort = '-createdAt'
@ -70,5 +76,6 @@ export {
setVideosSort,
setBlacklistSort,
setFollowersSort,
setFollowingSort
setFollowingSort,
setJobsSort
}

View File

@ -7,6 +7,7 @@ import { areValidationErrors } from './utils'
// Initialize constants here for better performances
const SORTABLE_USERS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.USERS)
const SORTABLE_JOBS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.JOBS)
const SORTABLE_VIDEO_ABUSES_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEO_ABUSES)
const SORTABLE_VIDEOS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.VIDEOS)
const SORTABLE_BLACKLISTS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.BLACKLISTS)
@ -15,6 +16,7 @@ const SORTABLE_FOLLOWERS_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.FOLLOW
const SORTABLE_FOLLOWING_COLUMNS = createSortableColumns(SORTABLE_COLUMNS.FOLLOWING)
const usersSortValidator = checkSort(SORTABLE_USERS_COLUMNS)
const jobsSortValidator = checkSort(SORTABLE_JOBS_COLUMNS)
const videoAbusesSortValidator = checkSort(SORTABLE_VIDEO_ABUSES_COLUMNS)
const videosSortValidator = checkSort(SORTABLE_VIDEOS_COLUMNS)
const blacklistSortValidator = checkSort(SORTABLE_BLACKLISTS_COLUMNS)
@ -31,7 +33,8 @@ export {
videosSortValidator,
blacklistSortValidator,
followersSortValidator,
followingSortValidator
followingSortValidator,
jobsSortValidator
}
// ---------------------------------------------------------------------------

View File

@ -1,12 +1,10 @@
import * as Sequelize from 'sequelize'
import * as Bluebird from 'bluebird'
// Don't use barrel, import just what we need
import { AccountInstance } from './account-interface'
import { User as FormattedUser } from '../../../shared/models/users/user.model'
import * as Sequelize from 'sequelize'
import { ResultList } from '../../../shared/models/result-list.model'
import { UserRight } from '../../../shared/models/users/user-right.enum'
import { UserRole } from '../../../shared/models/users/user-role'
import { User as FormattedUser } from '../../../shared/models/users/user.model'
import { AccountInstance } from './account-interface'
export namespace UserMethods {
export type HasRight = (this: UserInstance, right: UserRight) => boolean

View File

@ -1,18 +1,23 @@
import * as Bluebird from 'bluebird'
import * as Sequelize from 'sequelize'
import * as Promise from 'bluebird'
import { JobCategory, JobState } from '../../../shared/models/job.model'
import { Job as FormattedJob, JobCategory, JobState } from '../../../shared/models/job.model'
import { ResultList } from '../../../shared/models/result-list.model'
export namespace JobMethods {
export type ListWithLimitByCategory = (limit: number, state: JobState, category: JobCategory) => Promise<JobInstance[]>
export type ListWithLimitByCategory = (limit: number, state: JobState, category: JobCategory) => Bluebird<JobInstance[]>
export type ListForApi = (start: number, count: number, sort: string) => Bluebird< ResultList<JobInstance> >
export type ToFormattedJSON = (this: JobInstance) => FormattedJob
}
export interface JobClass {
listWithLimitByCategory: JobMethods.ListWithLimitByCategory
listForApi: JobMethods.ListForApi,
}
export interface JobAttributes {
state: JobState
category: JobCategory
handlerName: string
handlerInputData: any
}
@ -21,6 +26,8 @@ export interface JobInstance extends JobClass, JobAttributes, Sequelize.Instance
id: number
createdAt: Date
updatedAt: Date
toFormattedJSON: JobMethods.ToFormattedJSON
}
export interface JobModel extends JobClass, Sequelize.Model<JobInstance, JobAttributes> {}

View File

@ -1,19 +1,14 @@
import { values } from 'lodash'
import * as Sequelize from 'sequelize'
import { JOB_STATES, JOB_CATEGORIES } from '../../initializers'
import { addMethodsToModel } from '../utils'
import {
JobInstance,
JobAttributes,
JobMethods
} from './job-interface'
import { JobCategory, JobState } from '../../../shared/models/job.model'
import { JOB_CATEGORIES, JOB_STATES } from '../../initializers'
import { addMethodsToModel, getSort } from '../utils'
import { JobAttributes, JobInstance, JobMethods } from './job-interface'
let Job: Sequelize.Model<JobInstance, JobAttributes>
let listWithLimitByCategory: JobMethods.ListWithLimitByCategory
let listForApi: JobMethods.ListForApi
let toFormattedJSON: JobMethods.ToFormattedJSON
export default function defineJob (sequelize: Sequelize.Sequelize, DataTypes: Sequelize.DataTypes) {
Job = sequelize.define<JobInstance, JobAttributes>('Job',
@ -44,12 +39,30 @@ export default function defineJob (sequelize: Sequelize.Sequelize, DataTypes: Se
}
)
const classMethods = [ listWithLimitByCategory ]
addMethodsToModel(Job, classMethods)
const classMethods = [
listWithLimitByCategory,
listForApi
]
const instanceMethods = [
toFormattedJSON
]
addMethodsToModel(Job, classMethods, instanceMethods)
return Job
}
toFormattedJSON = function (this: JobInstance) {
return {
id: this.id,
state: this.state,
category: this.category,
handlerName: this.handlerName,
handlerInputData: this.handlerInputData,
createdAt: this.createdAt,
updatedAt: this.updatedAt
}
}
// ---------------------------------------------------------------------------
listWithLimitByCategory = function (limit: number, state: JobState, jobCategory: JobCategory) {
@ -66,3 +79,18 @@ listWithLimitByCategory = function (limit: number, state: JobState, jobCategory:
return Job.findAll(query)
}
listForApi = function (start: number, count: number, sort: string) {
const query = {
offset: start,
limit: count,
order: [ getSort(sort) ]
}
return Job.findAndCountAll(query).then(({ rows, count }) => {
return {
data: rows,
total: count
}
})
}

View File

@ -1,5 +1,6 @@
// Order of the tests we want to execute
import './follows'
import './jobs'
import './users'
import './services'
import './videos'

View File

@ -0,0 +1,84 @@
/* tslint:disable:no-unused-expression */
import 'mocha'
import * as request from 'supertest'
import { createUser, flushTests, getUserAccessToken, killallServers, runServer, ServerInfo, setAccessTokensToServers } from '../../utils'
describe('Test jobs API validators', function () {
const path = '/api/v1/jobs/'
let server: ServerInfo
let userAccessToken = ''
// ---------------------------------------------------------------
before(async function () {
this.timeout(120000)
await flushTests()
server = await runServer(1)
await setAccessTokensToServers([ server ])
const user = {
username: 'user1',
password: 'my super password'
}
await createUser(server.url, server.accessToken, user.username, user.password)
userAccessToken = await getUserAccessToken(server, user)
})
describe('When listing jobs', function () {
it('Should fail with a bad start pagination', async function () {
await request(server.url)
.get(path)
.query({ start: 'hello' })
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + server.accessToken)
.expect(400)
})
it('Should fail with a bad count pagination', async function () {
await request(server.url)
.get(path)
.query({ count: 'hello' })
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + server.accessToken)
.expect(400)
})
it('Should fail with an incorrect sort', async function () {
await request(server.url)
.get(path)
.query({ sort: 'hello' })
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + server.accessToken)
.expect(400)
})
it('Should fail with a non authenticated user', async function () {
await request(server.url)
.get(path)
.set('Accept', 'application/json')
.expect(401)
})
it('Should fail with a non admin user', async function () {
await request(server.url)
.get(path)
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + userAccessToken)
.expect(403)
})
})
after(async function () {
killallServers([ server ])
// Keep the logs if the test failed
if (this['ok']) {
await flushTests()
}
})
})

View File

@ -3,3 +3,4 @@
import './video-transcoder'
import './multiple-servers'
import './follows'
import './jobs'

64
server/tests/api/jobs.ts Normal file
View File

@ -0,0 +1,64 @@
/* tslint:disable:no-unused-expression */
import * as chai from 'chai'
import 'mocha'
import { flushTests, killallServers, ServerInfo, setAccessTokensToServers, wait } from '../utils'
import { doubleFollow } from '../utils/follows'
import { getJobsList, getJobsListPaginationAndSort } from '../utils/jobs'
import { flushAndRunMultipleServers } from '../utils/servers'
import { uploadVideo } from '../utils/videos'
import { dateIsValid } from '../utils/miscs'
const expect = chai.expect
describe('Test jobs', function () {
let servers: ServerInfo[]
before(async function () {
this.timeout(30000)
servers = await flushAndRunMultipleServers(2)
await setAccessTokensToServers(servers)
// Server 1 and server 2 follow each other
await doubleFollow(servers[0], servers[1])
})
it('Should create some jobs', async function () {
this.timeout(30000)
await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video1' })
await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' })
await wait(15000)
})
it('Should list jobs', async function () {
const res = await getJobsList(servers[1].url, servers[1].accessToken)
expect(res.body.total).to.be.above(2)
expect(res.body.data).to.have.length.above(2)
})
it('Should list jobs with sort and pagination', async function () {
const res = await getJobsListPaginationAndSort(servers[1].url, servers[1].accessToken, 4, 1, 'createdAt')
expect(res.body.total).to.be.above(2)
expect(res.body.data).to.have.lengthOf(1)
const job = res.body.data[0]
expect(job.state).to.equal('success')
expect(job.category).to.equal('transcoding')
expect(job.handlerName).to.have.length.above(3)
expect(dateIsValid(job.createdAt)).to.be.true
expect(dateIsValid(job.updatedAt)).to.be.true
})
after(async function () {
killallServers(servers)
// Keep the logs if the test failed
if (this['ok']) {
await flushTests()
}
})
})

View File

@ -61,7 +61,7 @@ async function doubleFollow (server1: ServerInfo, server2: ServerInfo) {
])
// Wait request propagation
await wait(20000)
await wait(10000)
return true
}

View File

@ -0,0 +1,33 @@
import * as request from 'supertest'
function getJobsList (url: string, accessToken: string) {
const path = '/api/v1/jobs'
return request(url)
.get(path)
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + accessToken)
.expect(200)
.expect('Content-Type', /json/)
}
function getJobsListPaginationAndSort (url: string, accessToken: string, start: number, count: number, sort: string) {
const path = '/api/v1/jobs'
return request(url)
.get(path)
.query({ start })
.query({ count })
.query({ sort })
.set('Accept', 'application/json')
.set('Authorization', 'Bearer ' + accessToken)
.expect(200)
.expect('Content-Type', /json/)
}
// ---------------------------------------------------------------------------
export {
getJobsList,
getJobsListPaginationAndSort
}

View File

@ -1,2 +1,12 @@
export type JobState = 'pending' | 'processing' | 'error' | 'success'
export type JobCategory = 'transcoding' | 'activitypub-http'
export interface Job {
id: number
state: JobState
category: JobCategory
handlerName: string
handlerInputData: any
createdAt: Date
updatedAt: Date
}

View File

@ -4,6 +4,7 @@ export enum UserRight {
MANAGE_SERVER_FOLLOW,
MANAGE_VIDEO_ABUSES,
MANAGE_VIDEO_BLACKLIST,
MANAGE_JOBS,
REMOVE_ANY_VIDEO,
REMOVE_ANY_VIDEO_CHANNEL
}

View File

@ -1,4 +1,5 @@
import { UserRight } from './user-right.enum'
import user from '../../../server/models/account/user'
// Keep the order
export enum UserRole {