From 566c125d6eee3bd907404523d94e1e0b5e403a46 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Dec 2019 14:14:01 +0100 Subject: [PATCH] Serve audit logs to client --- .../video-abuse-list.component.html | 2 +- .../video-blacklist-list.component.html | 2 +- .../+admin/system/jobs/jobs.component.html | 2 +- .../app/+admin/system/logs/log-row.model.ts | 19 ++ .../+admin/system/logs/logs.component.html | 13 +- .../+admin/system/logs/logs.component.scss | 10 + .../app/+admin/system/logs/logs.component.ts | 24 ++- .../app/+admin/system/logs/logs.service.ts | 20 +- .../users/user-list/user-list.component.html | 2 +- client/src/sass/primeng-custom.scss | 2 +- server/controllers/api/server/logs.ts | 57 +++++- server/helpers/audit-logger.ts | 3 +- server/helpers/logger.ts | 3 +- server/initializers/constants.ts | 4 + server/middlewares/validators/logs.ts | 19 +- server/tests/api/server/logs.ts | 173 +++++++++++------- shared/extra-utils/logs/logs.ts | 17 +- shared/models/server/log-level.type.ts | 2 +- 18 files changed, 287 insertions(+), 87 deletions(-) diff --git a/client/src/app/+admin/moderation/video-abuse-list/video-abuse-list.component.html b/client/src/app/+admin/moderation/video-abuse-list/video-abuse-list.component.html index 627437053..30eb2dbde 100644 --- a/client/src/app/+admin/moderation/video-abuse-list/video-abuse-list.component.html +++ b/client/src/app/+admin/moderation/video-abuse-list/video-abuse-list.component.html @@ -15,7 +15,7 @@ - + diff --git a/client/src/app/+admin/moderation/video-blacklist-list/video-blacklist-list.component.html b/client/src/app/+admin/moderation/video-blacklist-list/video-blacklist-list.component.html index 608dff2d8..a0b89acc6 100644 --- a/client/src/app/+admin/moderation/video-blacklist-list/video-blacklist-list.component.html +++ b/client/src/app/+admin/moderation/video-blacklist-list/video-blacklist-list.component.html @@ -15,7 +15,7 @@ - + diff --git a/client/src/app/+admin/system/jobs/jobs.component.html b/client/src/app/+admin/system/jobs/jobs.component.html index cd26257dd..de43b6448 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.html +++ b/client/src/app/+admin/system/jobs/jobs.component.html @@ -38,7 +38,7 @@ - + diff --git a/client/src/app/+admin/system/logs/log-row.model.ts b/client/src/app/+admin/system/logs/log-row.model.ts index 9bc7dafdd..b22581b5a 100644 --- a/client/src/app/+admin/system/logs/log-row.model.ts +++ b/client/src/app/+admin/system/logs/log-row.model.ts @@ -8,6 +8,10 @@ export class LogRow { message: string meta: string + by: string + domain: string + action: string + constructor (row: any) { this.date = new Date(row.timestamp) this.localeDate = this.date.toLocaleString() @@ -17,5 +21,20 @@ export class LogRow { const metaObj = omit(row, 'timestamp', 'level', 'message', 'label') if (Object.keys(metaObj).length !== 0) this.meta = JSON.stringify(metaObj, undefined, 2) + + if (row.level === 'audit') { + try { + const message = JSON.parse(row.message) + + this.by = message.user + this.domain = message.domain + this.action = message.action + + this.meta = JSON.stringify(message, null, 2) + this.message = '' + } catch (err) { + console.error('Cannot parse audit message.', err) + } + } } } diff --git a/client/src/app/+admin/system/logs/logs.component.html b/client/src/app/+admin/system/logs/logs.component.html index 45723a655..ddad1314f 100644 --- a/client/src/app/+admin/system/logs/logs.component.html +++ b/client/src/app/+admin/system/logs/logs.component.html @@ -1,11 +1,17 @@
+
+ +
+
-
+
@@ -23,9 +29,12 @@ [{{ log.localeDate }}] + By {{ log.by }} -> + {{ log.domain }} -> {{ log.action }} + {{ log.message }} - {{ log.meta }} + 
{{ log.meta }}
diff --git a/client/src/app/+admin/system/logs/logs.component.scss b/client/src/app/+admin/system/logs/logs.component.scss index 7ad2e853c..dae8b21c7 100644 --- a/client/src/app/+admin/system/logs/logs.component.scss +++ b/client/src/app/+admin/system/logs/logs.component.scss @@ -23,6 +23,10 @@ margin-right: 5px; } + .log-by { + margin: 0 5px; + } + .warn { color: $orange-color; } @@ -30,6 +34,12 @@ .error { color: $red; } + + pre { + margin-bottom: 5px; + white-space: pre-wrap; + word-wrap: break-word; + } } .header { diff --git a/client/src/app/+admin/system/logs/logs.component.ts b/client/src/app/+admin/system/logs/logs.component.ts index b2aca8461..b63f11953 100644 --- a/client/src/app/+admin/system/logs/logs.component.ts +++ b/client/src/app/+admin/system/logs/logs.component.ts @@ -17,9 +17,11 @@ export class LogsComponent implements OnInit { logs: LogRow[] = [] timeChoices: { id: string, label: string }[] = [] levelChoices: { id: LogLevel, label: string }[] = [] + logTypeChoices: { id: 'audit' | 'standard', label: string }[] = [] startDate: string level: LogLevel + logType: 'audit' | 'standard' constructor ( private logsService: LogsService, @@ -30,6 +32,7 @@ export class LogsComponent implements OnInit { ngOnInit (): void { this.buildTimeChoices() this.buildLevelChoices() + this.buildLogTypeChoices() this.load() } @@ -42,7 +45,7 @@ export class LogsComponent implements OnInit { load () { this.loading = true - this.logsService.getLogs(this.level, this.startDate) + this.logsService.getLogs({ isAuditLog: this.isAuditLog(), level: this.level, startDate: this.startDate }) .subscribe( logs => { this.logs = logs @@ -58,6 +61,10 @@ export class LogsComponent implements OnInit { ) } + isAuditLog () { + return this.logType === 'audit' + } + buildTimeChoices () { const lastHour = new Date() lastHour.setHours(lastHour.getHours() - 1) @@ -108,4 +115,19 @@ export class LogsComponent implements OnInit { this.level = 'warn' } + + buildLogTypeChoices () { + this.logTypeChoices = [ + { + id: 'standard', + label: this.i18n('Standard logs') + }, + { + id: 'audit', + label: this.i18n('Audit logs') + } + ] + + this.logType = 'audit' + } } diff --git a/client/src/app/+admin/system/logs/logs.service.ts b/client/src/app/+admin/system/logs/logs.service.ts index 24b9cb6d1..41b38c7ba 100644 --- a/client/src/app/+admin/system/logs/logs.service.ts +++ b/client/src/app/+admin/system/logs/logs.service.ts @@ -10,6 +10,7 @@ import { LogLevel } from '@shared/models/server/log-level.type' @Injectable() export class LogsService { private static BASE_LOG_URL = environment.apiUrl + '/api/v1/server/logs' + private static BASE_AUDIT_LOG_URL = environment.apiUrl + '/api/v1/server/audit-logs' constructor ( private authHttp: HttpClient, @@ -17,14 +18,25 @@ export class LogsService { private restExtractor: RestExtractor ) {} - getLogs (level: LogLevel, startDate: string, endDate?: string): Observable { + getLogs (options: { + isAuditLog: boolean, + startDate: string, + level?: LogLevel, + endDate?: string + }): Observable { + const { isAuditLog, startDate } = options + let params = new HttpParams() params = params.append('startDate', startDate) - params = params.append('level', level) - if (endDate) params.append('endDate', endDate) + if (!isAuditLog) params = params.append('level', options.level) + if (options.endDate) params.append('endDate', options.endDate) - return this.authHttp.get(LogsService.BASE_LOG_URL, { params }) + const path = isAuditLog + ? LogsService.BASE_AUDIT_LOG_URL + : LogsService.BASE_LOG_URL + + return this.authHttp.get(path, { params }) .pipe( map(rows => rows.map(r => new LogRow(r))), catchError(err => this.restExtractor.handleError(err)) diff --git a/client/src/app/+admin/users/user-list/user-list.component.html b/client/src/app/+admin/users/user-list/user-list.component.html index 822bb53da..885335313 100644 --- a/client/src/app/+admin/users/user-list/user-list.component.html +++ b/client/src/app/+admin/users/user-list/user-list.component.html @@ -49,7 +49,7 @@ - + diff --git a/client/src/sass/primeng-custom.scss b/client/src/sass/primeng-custom.scss index 6c3100746..0acffef3c 100644 --- a/client/src/sass/primeng-custom.scss +++ b/client/src/sass/primeng-custom.scss @@ -37,7 +37,7 @@ p-table { td { padding-left: 15px !important; - &:not(.action-cell) { + &:not(.action-cell):not(.expand-cell) { overflow: hidden !important; text-overflow: ellipsis !important; white-space: nowrap !important; diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts index e9d1f2efd..a0ca21cd5 100644 --- a/server/controllers/api/server/logs.ts +++ b/server/controllers/api/server/logs.ts @@ -3,11 +3,12 @@ import { UserRight } from '../../../../shared/models/users' import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' import { mtimeSortFilesDesc } from '../../../../shared/core-utils/logs/logs' import { readdir, readFile } from 'fs-extra' -import { MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants' +import { AUDIT_LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS, LOG_FILENAME } from '../../../initializers/constants' import { join } from 'path' -import { getLogsValidator } from '../../../middlewares/validators/logs' +import { getAuditLogsValidator, getLogsValidator } from '../../../middlewares/validators/logs' import { LogLevel } from '../../../../shared/models/server/log-level.type' import { CONFIG } from '../../../initializers/config' +import { logger } from '@server/helpers/logger' const logsRouter = express.Router() @@ -18,6 +19,13 @@ logsRouter.get('/logs', asyncMiddleware(getLogs) ) +logsRouter.get('/audit-logs', + authenticate, + ensureUserHasRight(UserRight.MANAGE_LOGS), + getAuditLogsValidator, + asyncMiddleware(getAuditLogs) +) + // --------------------------------------------------------------------------- export { @@ -26,18 +34,50 @@ export { // --------------------------------------------------------------------------- +const auditLogNameFilter = generateLogNameFilter(AUDIT_LOG_FILENAME) +async function getAuditLogs (req: express.Request, res: express.Response) { + const output = await generateOutput({ + startDateQuery: req.query.startDate, + endDateQuery: req.query.endDate, + level: 'audit', + nameFilter: auditLogNameFilter + }) + + return res.json(output).end() +} + +const logNameFilter = generateLogNameFilter(LOG_FILENAME) async function getLogs (req: express.Request, res: express.Response) { + const output = await generateOutput({ + startDateQuery: req.query.startDate, + endDateQuery: req.query.endDate, + level: req.query.level || 'info', + nameFilter: logNameFilter + }) + + return res.json(output).end() +} + +async function generateOutput (options: { + startDateQuery: string, + endDateQuery?: string, + level: LogLevel, + nameFilter: RegExp +}) { + const { startDateQuery, level, nameFilter } = options + const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR) let currentSize = 0 - const startDate = new Date(req.query.startDate) - const endDate = req.query.endDate ? new Date(req.query.endDate) : new Date() - const level: LogLevel = req.query.level || 'info' + const startDate = new Date(startDateQuery) + const endDate = options.endDateQuery ? new Date(options.endDateQuery) : new Date() let output: string[] = [] for (const meta of sortedLogFiles) { + if (nameFilter.exec(meta.file) === null) continue + const path = join(CONFIG.STORAGE.LOG_DIR, meta.file) const result = await getOutputFromFile(path, startDate, endDate, level, currentSize) @@ -49,7 +89,7 @@ async function getLogs (req: express.Request, res: express.Response) { if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS || (result.logTime && result.logTime < startDate.getTime())) break } - return res.json(output).end() + return output } async function getOutputFromFile (path: string, startDate: Date, endDate: Date, level: LogLevel, currentSize: number) { @@ -58,6 +98,7 @@ async function getOutputFromFile (path: string, startDate: Date, endDate: Date, let logTime: number const logsLevel: { [ id in LogLevel ]: number } = { + audit: -1, debug: 0, info: 1, warn: 2, @@ -93,3 +134,7 @@ async function getOutputFromFile (path: string, startDate: Date, endDate: Date, return { currentSize, output: output.reverse(), logTime } } + +function generateLogNameFilter (baseName: string) { + return new RegExp('^' + baseName.replace(/\.log$/, '') + '\d*.log$') +} diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index f536da439..9b258dc3a 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts @@ -9,6 +9,7 @@ import { User, VideoAbuse, VideoChannel, VideoDetails, VideoImport } from '../.. import { VideoComment } from '../../shared/models/videos/video-comment.model' import { CustomConfig } from '../../shared/models/server/custom-config.model' import { CONFIG } from '../initializers/config' +import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' function getAuditIdFromRes (res: express.Response) { return res.locals.oauth.token.User.username @@ -29,7 +30,7 @@ const auditLogger = winston.createLogger({ levels: { audit: 0 }, transports: [ new winston.transports.File({ - filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube-audit.log'), + filename: path.join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME), level: 'audit', maxsize: 5242880, maxFiles: 5, diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index d21746963..c2ff2bae6 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -5,6 +5,7 @@ import * as winston from 'winston' import { FileTransportOptions } from 'winston/lib/winston/transports' import { CONFIG } from '../initializers/config' import { omit } from 'lodash' +import { LOG_FILENAME } from '@server/initializers/constants' const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT @@ -58,7 +59,7 @@ const labelFormatter = winston.format.label({ }) const fileLoggerOptions: FileTransportOptions = { - filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube.log'), + filename: path.join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME), handleExceptions: true, format: winston.format.combine( winston.format.timestamp(), diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index af70e7b88..bdabe7f66 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -603,6 +603,8 @@ const FEEDS = { } const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000 +const LOG_FILENAME = 'peertube.log' +const AUDIT_LOG_FILENAME = 'peertube-audit.log' // --------------------------------------------------------------------------- @@ -684,6 +686,7 @@ export { BCRYPT_SALT_SIZE, TRACKER_RATE_LIMITS, FILES_CACHE, + LOG_FILENAME, CONSTRAINTS_FIELDS, EMBED_SIZE, REDUNDANCY, @@ -693,6 +696,7 @@ export { OAUTH_LIFETIME, CUSTOM_HTML_TAG_COMMENTS, BROADCAST_CONCURRENCY, + AUDIT_LOG_FILENAME, PAGINATION, ACTOR_FOLLOW_SCORE, PREVIEWS_SIZE, diff --git a/server/middlewares/validators/logs.ts b/server/middlewares/validators/logs.ts index 07f3f552f..70e4d0d99 100644 --- a/server/middlewares/validators/logs.ts +++ b/server/middlewares/validators/logs.ts @@ -24,8 +24,25 @@ const getLogsValidator = [ } ] +const getAuditLogsValidator = [ + query('startDate') + .custom(isDateValid).withMessage('Should have a valid start date'), + query('endDate') + .optional() + .custom(isDateValid).withMessage('Should have a valid end date'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking getAuditLogsValidator parameters.', { parameters: req.query }) + + if (areValidationErrors(req, res)) return + + return next() + } +] + // --------------------------------------------------------------------------- export { - getLogsValidator + getLogsValidator, + getAuditLogsValidator } diff --git a/server/tests/api/server/logs.ts b/server/tests/api/server/logs.ts index 68f442199..d3c877408 100644 --- a/server/tests/api/server/logs.ts +++ b/server/tests/api/server/logs.ts @@ -2,17 +2,10 @@ import * as chai from 'chai' import 'mocha' -import { - flushTests, - killallServers, - flushAndRunServer, - ServerInfo, - setAccessTokensToServers, - cleanupTests -} from '../../../../shared/extra-utils/index' +import { cleanupTests, flushAndRunServer, ServerInfo, setAccessTokensToServers } from '../../../../shared/extra-utils/index' import { waitJobs } from '../../../../shared/extra-utils/server/jobs' import { uploadVideo } from '../../../../shared/extra-utils/videos/videos' -import { getLogs } from '../../../../shared/extra-utils/logs/logs' +import { getAuditLogs, getLogs } from '../../../../shared/extra-utils/logs/logs' const expect = chai.expect @@ -26,69 +19,123 @@ describe('Test logs', function () { await setAccessTokensToServers([ server ]) }) - it('Should get logs with a start date', async function () { - this.timeout(10000) + describe('With the standard log file', function () { + it('Should get logs with a start date', async function () { + this.timeout(10000) - await uploadVideo(server.url, server.accessToken, { name: 'video 1' }) - await waitJobs([ server ]) + await uploadVideo(server.url, server.accessToken, { name: 'video 1' }) + await waitJobs([ server ]) - const now = new Date() + const now = new Date() - await uploadVideo(server.url, server.accessToken, { name: 'video 2' }) - await waitJobs([ server ]) + await uploadVideo(server.url, server.accessToken, { name: 'video 2' }) + await waitJobs([ server ]) - const res = await getLogs(server.url, server.accessToken, now) - const logsString = JSON.stringify(res.body) - - expect(logsString.includes('video 1')).to.be.false - expect(logsString.includes('video 2')).to.be.true - }) - - it('Should get logs with an end date', async function () { - this.timeout(20000) - - await uploadVideo(server.url, server.accessToken, { name: 'video 3' }) - await waitJobs([ server ]) - - const now1 = new Date() - - await uploadVideo(server.url, server.accessToken, { name: 'video 4' }) - await waitJobs([ server ]) - - const now2 = new Date() - - await uploadVideo(server.url, server.accessToken, { name: 'video 5' }) - await waitJobs([ server ]) - - const res = await getLogs(server.url, server.accessToken, now1, now2) - const logsString = JSON.stringify(res.body) - - expect(logsString.includes('video 3')).to.be.false - expect(logsString.includes('video 4')).to.be.true - expect(logsString.includes('video 5')).to.be.false - }) - - it('Should get filter by level', async function () { - this.timeout(10000) - - const now = new Date() - - await uploadVideo(server.url, server.accessToken, { name: 'video 6' }) - await waitJobs([ server ]) - - { - const res = await getLogs(server.url, server.accessToken, now, undefined, 'info') + const res = await getLogs(server.url, server.accessToken, now) const logsString = JSON.stringify(res.body) - expect(logsString.includes('video 6')).to.be.true - } + expect(logsString.includes('video 1')).to.be.false + expect(logsString.includes('video 2')).to.be.true + }) - { - const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn') + it('Should get logs with an end date', async function () { + this.timeout(20000) + + await uploadVideo(server.url, server.accessToken, { name: 'video 3' }) + await waitJobs([ server ]) + + const now1 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 4' }) + await waitJobs([ server ]) + + const now2 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 5' }) + await waitJobs([ server ]) + + const res = await getLogs(server.url, server.accessToken, now1, now2) const logsString = JSON.stringify(res.body) - expect(logsString.includes('video 6')).to.be.false - } + expect(logsString.includes('video 3')).to.be.false + expect(logsString.includes('video 4')).to.be.true + expect(logsString.includes('video 5')).to.be.false + }) + + it('Should get filter by level', async function () { + this.timeout(10000) + + const now = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 6' }) + await waitJobs([ server ]) + + { + const res = await getLogs(server.url, server.accessToken, now, undefined, 'info') + const logsString = JSON.stringify(res.body) + + expect(logsString.includes('video 6')).to.be.true + } + + { + const res = await getLogs(server.url, server.accessToken, now, undefined, 'warn') + const logsString = JSON.stringify(res.body) + + expect(logsString.includes('video 6')).to.be.false + } + }) + }) + + describe('With the audit log', function () { + it('Should get logs with a start date', async function () { + this.timeout(10000) + + await uploadVideo(server.url, server.accessToken, { name: 'video 7' }) + await waitJobs([ server ]) + + const now = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 8' }) + await waitJobs([ server ]) + + const res = await getAuditLogs(server.url, server.accessToken, now) + const logsString = JSON.stringify(res.body) + + expect(logsString.includes('video 7')).to.be.false + expect(logsString.includes('video 8')).to.be.true + + expect(res.body).to.have.lengthOf(1) + + const item = res.body[0] + + const message = JSON.parse(item.message) + expect(message.domain).to.equal('videos') + expect(message.action).to.equal('create') + }) + + it('Should get logs with an end date', async function () { + this.timeout(20000) + + await uploadVideo(server.url, server.accessToken, { name: 'video 9' }) + await waitJobs([ server ]) + + const now1 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 10' }) + await waitJobs([ server ]) + + const now2 = new Date() + + await uploadVideo(server.url, server.accessToken, { name: 'video 11' }) + await waitJobs([ server ]) + + const res = await getAuditLogs(server.url, server.accessToken, now1, now2) + const logsString = JSON.stringify(res.body) + + expect(logsString.includes('video 9')).to.be.false + expect(logsString.includes('video 10')).to.be.true + expect(logsString.includes('video 11')).to.be.false + }) }) after(async function () { diff --git a/shared/extra-utils/logs/logs.ts b/shared/extra-utils/logs/logs.ts index cbb1afb93..c494c1f1e 100644 --- a/shared/extra-utils/logs/logs.ts +++ b/shared/extra-utils/logs/logs.ts @@ -13,6 +13,19 @@ function getLogs (url: string, accessToken: string, startDate: Date, endDate?: D }) } -export { - getLogs +function getAuditLogs (url: string, accessToken: string, startDate: Date, endDate?: Date) { + const path = '/api/v1/server/audit-logs' + + return makeGetRequest({ + url, + path, + token: accessToken, + query: { startDate, endDate }, + statusCodeExpected: 200 + }) +} + +export { + getLogs, + getAuditLogs } diff --git a/shared/models/server/log-level.type.ts b/shared/models/server/log-level.type.ts index ce91559e3..4afb92d11 100644 --- a/shared/models/server/log-level.type.ts +++ b/shared/models/server/log-level.type.ts @@ -1 +1 @@ -export type LogLevel = 'debug' | 'info' | 'warn' | 'error' +export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'audit'